From a7e7fa4efef34db758e107d6a9fa2a060e64e6c7 Mon Sep 17 00:00:00 2001
From: Hao Jin <hjjn.amzn@gmail.com>
Date: Thu, 16 Apr 2020 18:47:06 -0700
Subject: [PATCH 01/44] add zero grad for npi_unique (#18080)

---
 src/operator/numpy/np_unique_op.cc     | 1 +
 tests/python/unittest/test_numpy_op.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/operator/numpy/np_unique_op.cc b/src/operator/numpy/np_unique_op.cc
index 2f57733a72b2..7a299cdd5221 100644
--- a/src/operator/numpy/np_unique_op.cc
+++ b/src/operator/numpy/np_unique_op.cc
@@ -375,6 +375,7 @@ NNVM_REGISTER_OP(_npi_unique)
   [](const NodeAttrs& attrs) {
     return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
   })
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
 .add_argument("data", "NDArray-or-Symbol", "The input array")
 .add_arguments(NumpyUniqueParam::__FIELDS__());
 
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 15e9bd4e4b9a..8a7d8f030ac3 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -6549,7 +6549,7 @@ def hybrid_forward(self, F, a):
         ((5, 3, 4), True, True, True, 1),
     ]
     for dtype in ['float32', 'float64', 'int8', 'uint8', 'int32', 'int64']:
-        for hybridize in [False]:
+        for hybridize in [False, True]:
             for config in configs:
                 test_unique = TestUnique(*config[1:])
                 if hybridize:

From b9c6a4a9e29c056411f26b4364a12ecd622b747c Mon Sep 17 00:00:00 2001
From: Yiyan66 <57363390+Yiyan66@users.noreply.github.com>
Date: Thu, 27 Feb 2020 05:44:55 +0800
Subject: [PATCH 02/44] flatnonzero (#17690)

---
 python/mxnet/ndarray/numpy/_op.py             | 41 ++++++++++++++++++-
 python/mxnet/numpy/fallback.py                |  2 -
 python/mxnet/numpy/multiarray.py              | 41 ++++++++++++++++++-
 python/mxnet/numpy_dispatch_protocol.py       |  1 +
 python/mxnet/symbol/numpy/_symbol.py          | 28 ++++++++++++-
 .../unittest/test_numpy_interoperability.py   |  6 ++-
 tests/python/unittest/test_numpy_op.py        | 30 ++++++++++++++
 7 files changed, 142 insertions(+), 7 deletions(-)

diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 39a307bec9e1..82b57fb8cc1f 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -38,7 +38,7 @@
            'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_',
+           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
@@ -4989,6 +4989,45 @@ def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-
         raise NotImplementedError('Do not support column-major (Fortran-style) order at this moment')
 
 
+def flatnonzero(a):
+    r"""
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to np.nonzero(np.ravel(a))[0].
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+    """
+    return nonzero(ravel(a))[0]
+
+
 def diag_indices_from(arr):
     """
     This returns a tuple of indices that can be used to access the main diagonal of an array
diff --git a/python/mxnet/numpy/fallback.py b/python/mxnet/numpy/fallback.py
index 1e45d8e54cc2..b98d377e4cd2 100644
--- a/python/mxnet/numpy/fallback.py
+++ b/python/mxnet/numpy/fallback.py
@@ -38,7 +38,6 @@
     'digitize',
     'divmod',
     'extract',
-    'flatnonzero',
     'float_power',
     'frexp',
     'heaviside',
@@ -124,7 +123,6 @@
 digitize = onp.digitize
 divmod = onp.divmod
 extract = onp.extract
-flatnonzero = onp.flatnonzero
 float_power = onp.float_power
 frexp = onp.frexp
 heaviside = onp.heaviside
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index fceaaf3a282f..3ff984c7a2a3 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -57,7 +57,7 @@
            'degrees', 'log2', 'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'histogram',
            'fix', 'ceil', 'floor', 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'append', 'argsort',
            'sort', 'tensordot', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange',
-           'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
+           'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'flatnonzero',
            'concatenate', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
            'average', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'insert',
            'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman',
@@ -6837,6 +6837,45 @@ def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-
     return _mx_nd_np.unravel_index(indices, shape, order=order)
 
 
+def flatnonzero(a):
+    r"""
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to np.nonzero(np.ravel(a))[0].
+
+    Parameters
+    ----------
+    a : array_like
+        Input data.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+
+    Examples
+    --------
+    >>> x = np.arange(-2, 3)
+    >>> x
+    array([-2, -1,  0,  1,  2])
+    >>> np.flatnonzero(x)
+    array([0, 1, 3, 4])
+
+    Use the indices of the non-zero elements as an index array to extract
+    these elements:
+
+    >>> x.ravel()[np.flatnonzero(x)]
+    array([-2, -1,  1,  2])
+    """
+    return _mx_nd_np.flatnonzero(a)
+
+
 def diag_indices_from(arr):
     """
     This returns a tuple of indices that can be used to access the main diagonal of an array
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index d8d7c0907bcf..781ec55b3796 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -142,6 +142,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'transpose',
     'unique',
     'unravel_index',
+    'flatnonzero',
     'diag_indices_from',
     'delete',
     'var',
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index da8d5d738f46..8756b6a78ac9 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -43,7 +43,7 @@
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_',
+           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
@@ -4664,6 +4664,32 @@ def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-
         raise NotImplementedError('Don not support column-major (Fortran-style) order at this moment')
 
 
+def flatnonzero(a):
+    r"""
+    Return indices that are non-zero in the flattened version of a.
+
+    This is equivalent to np.nonzero(np.ravel(a))[0].
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input data.
+
+    Returns
+    -------
+    res : _Symbol
+        Output array, containing the indices of the elements of `a.ravel()`
+        that are non-zero.
+
+    See Also
+    --------
+    nonzero : Return the indices of the non-zero elements of the input array.
+    ravel : Return a 1-D array containing the elements of the input array.
+    """
+    out = _npi.nonzero(ravel(a))
+    return out.reshape(-1,)
+
+
 def diag_indices_from(arr):
     """
     This returns a tuple of indices that can be used to access the main diagonal of an array
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 18b26579f740..298d565dc237 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -2222,8 +2222,10 @@ def _add_workload_extract():
     OpArgMngr.add_workload('extract', condition, arr)
 
 
-def _add_workload_flatnonzero():
+def _add_workload_flatnonzero(array_pool):
     x = np.array([-2, -1,  0,  1,  2])
+    OpArgMngr.add_workload('flatnonzero', array_pool['4x1'])
+    OpArgMngr.add_workload('flatnonzero', array_pool['1x2'])
     OpArgMngr.add_workload('flatnonzero', x)
 
 
@@ -2911,7 +2913,7 @@ def _prepare_workloads():
     _add_workload_digitize()
     _add_workload_divmod()
     _add_workload_extract()
-    _add_workload_flatnonzero()
+    _add_workload_flatnonzero(array_pool)
     _add_workload_float_power()
     _add_workload_frexp()
     _add_workload_histogram2d()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 8a7d8f030ac3..9cfd6d6c926e 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -6456,6 +6456,36 @@ def hybrid_forward(self, F, x):
                     assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol)
 
 
+@with_seed()
+@use_np
+def test_np_flatnonzero():
+    class TestFlatnonzero(HybridBlock):
+        def __init__(self):
+            super(TestFlatnonzero, self).__init__()
+
+        def hybrid_forward(self, F, a):
+            return F.np.flatnonzero(a)
+
+    shapes = [(1,), (4, 3), (4, 5), (2, 1), (6, 5, 6), (4, 2, 1, 2),
+              (5, 1, 3, 3), (3, 3, 1, 0),]
+    types = ['int32', 'int64', 'float32', 'float64']
+    hybridizes = [True, False]
+    for hybridize, oneType, shape in itertools.product(hybridizes, types, shapes):
+        rtol, atol = 1e-3, 1e-5
+        test_flatnonzero = TestFlatnonzero()
+        if hybridize:
+            test_flatnonzero.hybridize()
+        x = rand_ndarray(shape, dtype=oneType).as_np_ndarray()
+        np_out = _np.flatnonzero(x.asnumpy())
+        mx_out = test_flatnonzero(x)
+        assert mx_out.shape == np_out.shape
+        assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol)
+
+        mx_out = np.flatnonzero(x)
+        np_out = _np.flatnonzero(x.asnumpy())
+        assert_almost_equal(mx_out.asnumpy(), np_out, rtol=rtol, atol=atol)
+
+
 @with_seed()
 @use_np
 def test_np_round():

From 8b5cb5efe90dad5afa15d7a5ac9f40263a937a8b Mon Sep 17 00:00:00 2001
From: Hao Jin <hjjn.amzn@gmail.com>
Date: Wed, 4 Mar 2020 22:06:46 -0800
Subject: [PATCH 03/44] [Numpy] FFI for cumsum and add (#17747)

* FFI cumsum

* Dispatch ufunc

* Add PythonArg

* Remove unused data type

* Seperate op_utils and utils
---
 include/mxnet/runtime/c_runtime_api.h         |  13 +--
 include/mxnet/runtime/packed_func.h           |  36 +-----
 include/mxnet/runtime/py_arg.h                |  42 +++++++
 python/mxnet/_ffi/_ctypes/function.py         |   7 +-
 python/mxnet/_ffi/_ctypes/types.py            |  16 ++-
 python/mxnet/_ffi/_cython/base.pxi            |  13 +--
 python/mxnet/_ffi/_cython/function.pxi        |  18 ++-
 python/mxnet/_numpy_op_doc.py                 |  51 --------
 python/mxnet/ndarray/numpy/_op.py             |  58 ++++++++-
 python/mxnet/numpy/multiarray.py              |  57 ++++++++-
 python/mxnet/symbol/numpy/_symbol.py          |  39 ++++++-
 src/api/operator/numpy/np_cumsum.cc           |  67 +++++++++++
 .../numpy/np_elemwise_broadcast_op.cc         |  39 +++++++
 src/api/operator/numpy/np_init_op.cc          |   5 +-
 src/api/operator/numpy/np_tensordot_op.cc     |  11 +-
 src/api/operator/op_utils.cc                  |  55 +++++++++
 src/api/operator/op_utils.h                   |  35 ++++++
 src/api/operator/ufunc_helper.cc              | 110 ++++++++++++++++++
 src/api/operator/ufunc_helper.h               |  36 ++++++
 src/api/operator/utils.cc                     |  22 ++++
 src/api/operator/utils.h                      |  21 +---
 src/operator/numpy/np_cumsum-inl.h            |  13 +++
 src/operator/numpy/np_cumsum.cc               |   7 +-
 src/operator/numpy/np_cumsum.cu               |   4 +-
 24 files changed, 629 insertions(+), 146 deletions(-)
 create mode 100644 include/mxnet/runtime/py_arg.h
 create mode 100644 src/api/operator/numpy/np_cumsum.cc
 create mode 100644 src/api/operator/numpy/np_elemwise_broadcast_op.cc
 create mode 100644 src/api/operator/op_utils.cc
 create mode 100644 src/api/operator/op_utils.h
 create mode 100644 src/api/operator/ufunc_helper.cc
 create mode 100644 src/api/operator/ufunc_helper.h

diff --git a/include/mxnet/runtime/c_runtime_api.h b/include/mxnet/runtime/c_runtime_api.h
index 208a64326ac4..bbc8862d5439 100644
--- a/include/mxnet/runtime/c_runtime_api.h
+++ b/include/mxnet/runtime/c_runtime_api.h
@@ -47,14 +47,11 @@ typedef enum {
   kNull = 4U,
   kMXNetType = 5U,
   kMXNetContext = 6U,
-  kArrayHandle = 7U,
-  kObjectHandle = 8U,
-  kModuleHandle = 9U,
-  kFuncHandle = 10U,
-  kStr = 11U,
-  kBytes = 12U,
-  kNDArrayContainer = 13U,
-  kNDArrayHandle = 14U,
+  kObjectHandle = 7U,
+  kStr = 8U,
+  kBytes = 9U,
+  kPyArg = 10U,
+  kNDArrayHandle = 11U,
   // Extension codes for other frameworks to integrate MXNet PackedFunc.
   // To make sure each framework's id do not conflict, use first and
   // last sections to mark ranges.
diff --git a/include/mxnet/runtime/packed_func.h b/include/mxnet/runtime/packed_func.h
index 16351a7604dc..ac7b462ce471 100644
--- a/include/mxnet/runtime/packed_func.h
+++ b/include/mxnet/runtime/packed_func.h
@@ -32,6 +32,7 @@
 #include <mxnet/runtime/container.h>
 #include <mxnet/runtime/ffi_helper.h>
 #include <mxnet/runtime/data_type.h>
+#include <mxnet/runtime/py_arg.h>
 #include <mxnet/node/container.h>
 #include <mxnet/ir/expr.h>
 #include <mxnet/ndarray.h>
@@ -416,7 +417,6 @@ class MXNetPODValue_ {
   }
   operator void*() const {
     if (type_code_ == kNull) return nullptr;
-    if (type_code_ == kArrayHandle) return value_.v_handle;
     MXNET_CHECK_TYPE_CODE(type_code_, kHandle);
     return value_.v_handle;
   }
@@ -520,11 +520,6 @@ class MXNetArgValue : public MXNetPODValue_ {
     MXNET_CHECK_TYPE_CODE(type_code_, kNDArrayHandle);
     return reinterpret_cast<::mxnet::NDArray*>(value_.v_handle);
   }
-  operator PackedFunc() const {
-    if (type_code_ == kNull) return PackedFunc();
-    MXNET_CHECK_TYPE_CODE(type_code_, kFuncHandle);
-    return *ptr<PackedFunc>();
-  }
   template<typename FType>
   operator TypedPackedFunc<FType>() const {
     return TypedPackedFunc<FType>(operator PackedFunc());
@@ -597,11 +592,6 @@ class MXNetRetValue : public MXNetPODValue_ {
   operator MXNetDataType() const {
     return MXNetDataType(operator DLDataType());
   }
-  operator PackedFunc() const {
-    if (type_code_ == kNull) return PackedFunc();
-    MXNET_CHECK_TYPE_CODE(type_code_, kFuncHandle);
-    return *ptr<PackedFunc>();
-  }
   template<typename FType>
   operator TypedPackedFunc<FType>() const {
     return TypedPackedFunc<FType>(operator PackedFunc());
@@ -668,10 +658,6 @@ class MXNetRetValue : public MXNetPODValue_ {
     SwitchToObject(kObjectHandle, std::move(other));
     return *this;
   }
-  MXNetRetValue& operator=(PackedFunc f) {
-    this->SwitchToClass(kFuncHandle, f);
-    return *this;
-  }
   template<typename FType>
   MXNetRetValue& operator=(const TypedPackedFunc<FType>& f) {
     return operator=(f.packed());
@@ -689,6 +675,11 @@ class MXNetRetValue : public MXNetPODValue_ {
     value_.v_handle = reinterpret_cast<void*>(value);
     return *this;
   }
+  MXNetRetValue& operator=(const PythonArg& value) {
+    this->SwitchToPOD(kPyArg);
+    value_.v_int64 = value.offset();
+    return *this;
+  }
   template<typename T,
            typename = typename std::enable_if<
              extension_type_info<T>::code != 0>::type>
@@ -717,7 +708,6 @@ class MXNetRetValue : public MXNetPODValue_ {
   /*! \return The value field, if the data is POD */
   const MXNetValue& value() const {
     CHECK(type_code_ != kObjectHandle &&
-          type_code_ != kFuncHandle &&
           type_code_ != kStr) << "MXNetRetValue.value can only be used for POD data";
     return value_;
   }
@@ -741,10 +731,6 @@ class MXNetRetValue : public MXNetPODValue_ {
         SwitchToClass<std::string>(kBytes, other);
         break;
       }
-      case kFuncHandle: {
-        SwitchToClass<PackedFunc>(kFuncHandle, other);
-        break;
-      }
       case kObjectHandle: {
         *this = other.operator ObjectRef();
         break;
@@ -792,7 +778,6 @@ class MXNetRetValue : public MXNetPODValue_ {
     if (type_code_ == kNull) return;
     switch (type_code_) {
       case kStr: delete ptr<std::string>(); break;
-      case kFuncHandle: delete ptr<PackedFunc>(); break;
       case kObjectHandle: {
         static_cast<Object*>(value_.v_handle)->DecRef();
         break;
@@ -857,7 +842,6 @@ inline const char* TypeCode2Str(int type_code) {
     case kBytes: return "bytes";
     case kHandle: return "handle";
     case kNull: return "NULL";
-    case kFuncHandle: return "FunctionHandle";
     case kObjectHandle: return "ObjectCell";
     default: LOG(FATAL) << "unknown type_code="
                         << static_cast<int>(type_code); return "";
@@ -1012,10 +996,6 @@ class MXNetArgsSetter {
     values_[i].v_handle = value;
     type_codes_[i] = kHandle;
   }
-  void operator()(size_t i, DLTensor* value) const {
-    values_[i].v_handle = value;
-    type_codes_[i] = kArrayHandle;
-  }
   void operator()(size_t i, const char* value) const {
     values_[i].v_str = value;
     type_codes_[i] = kStr;
@@ -1038,10 +1018,6 @@ class MXNetArgsSetter {
     values_[i].v_handle = const_cast<MXNetByteArray*>(&value);
     type_codes_[i] = kBytes;
   }
-  void operator()(size_t i, const PackedFunc& value) const {  // NOLINT(*)
-    values_[i].v_handle = const_cast<PackedFunc*>(&value);
-    type_codes_[i] = kFuncHandle;
-  }
   template<typename FType>
   void operator()(size_t i, const TypedPackedFunc<FType>& value) const {  // NOLINT(*)
     operator()(i, value.packed());
diff --git a/include/mxnet/runtime/py_arg.h b/include/mxnet/runtime/py_arg.h
new file mode 100644
index 000000000000..81d1b30a573e
--- /dev/null
+++ b/include/mxnet/runtime/py_arg.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * \file py_arg.h
+ * \brief Python runtime arguments specifier.
+ */
+#ifndef MXNET_RUNTIME_PY_ARG_H_
+#define MXNET_RUNTIME_PY_ARG_H_
+
+namespace mxnet {
+namespace runtime {
+
+class PythonArg {
+ public:
+  explicit PythonArg(int offset): offset_(offset) {}
+  int offset() const {
+    return offset_;
+  }
+ private:
+  int offset_;
+};
+
+}  // namespace runtime
+
+}  // namespace mxnet
+#endif  //  MXNET_RUNTIME_PY_ARG_H_
diff --git a/python/mxnet/_ffi/_ctypes/function.py b/python/mxnet/_ffi/_ctypes/function.py
index 5b126913b998..0a005dd7b749 100644
--- a/python/mxnet/_ffi/_ctypes/function.py
+++ b/python/mxnet/_ffi/_ctypes/function.py
@@ -22,6 +22,7 @@
 """
 import ctypes
 from numbers import Number, Integral
+import numpy as onp
 
 from ...base import get_last_ffi_error, _LIB
 from ..base import c_str
@@ -66,6 +67,9 @@ def _make_mxnet_args(args, temp_args):
         elif isinstance(arg, ctypes.c_void_p):
             values[i].v_handle = arg
             type_codes[i] = TypeCode.HANDLE
+        elif isinstance(arg, type):
+            values[i].v_str = c_str(onp.dtype(arg).name)
+            type_codes[i] = TypeCode.STR
         else:
             raise TypeError("Don't know how to handle type %s" % type(arg))
     return values, type_codes, num_args
@@ -110,7 +114,8 @@ def __call__(self, *args):
             raise get_last_ffi_error()
         _ = temp_args
         _ = args
-        return RETURN_SWITCH[ret_tcode.value](ret_val)
+        return (RETURN_SWITCH[ret_tcode.value](ret_val) if ret_tcode.value != TypeCode.PYARG
+                else RETURN_SWITCH[ret_tcode.value](ret_val, args))
 
 
 _CLASS_OBJECT = None
diff --git a/python/mxnet/_ffi/_ctypes/types.py b/python/mxnet/_ffi/_ctypes/types.py
index 265408e5ba93..d1b253af2da3 100644
--- a/python/mxnet/_ffi/_ctypes/types.py
+++ b/python/mxnet/_ffi/_ctypes/types.py
@@ -32,14 +32,11 @@ class TypeCode(object):
     NULL = 4
     MXNET_TYPE = 5
     MXNET_CONTEXT = 6
-    ARRAY_HANDLE = 7
-    OBJECT_HANDLE = 8
-    MODULE_HANDLE = 9
-    FUNC_HANDLE = 10
-    STR = 11
-    BYTES = 12
-    NDARRAY_CONTAINER = 13
-    NDARRAYHANDLE = 14
+    OBJECT_HANDLE = 7
+    STR = 8
+    BYTES = 9
+    PYARG = 10
+    NDARRAYHANDLE = 11
     EXT_BEGIN = 15
 
 
@@ -54,5 +51,6 @@ class MXNetValue(ctypes.Union):
     TypeCode.INT: lambda x: x.v_int64,
     TypeCode.FLOAT: lambda x: x.v_float64,
     TypeCode.NULL: lambda x: None,
-    TypeCode.NDARRAYHANDLE: lambda x: _global_var._np_ndarray_cls(handle=NDArrayHandle(x.v_handle))
+    TypeCode.NDARRAYHANDLE: lambda x: _global_var._np_ndarray_cls(handle=NDArrayHandle(x.v_handle)),
+    TypeCode.PYARG: lambda x, args: args[x.v_int64],
 }
diff --git a/python/mxnet/_ffi/_cython/base.pxi b/python/mxnet/_ffi/_cython/base.pxi
index 1c393e8e241c..bc2273bacd0d 100644
--- a/python/mxnet/_ffi/_cython/base.pxi
+++ b/python/mxnet/_ffi/_cython/base.pxi
@@ -32,14 +32,11 @@ cdef enum MXNetTypeCode:
     kNull = 4
     kMXNetType = 5
     kMXNetContext = 6
-    kArrayHandle = 7
-    kObjectHandle = 8
-    kModuleHandle = 9
-    kFuncHandle = 10
-    kStr = 11
-    kBytes = 12
-    kNDArrayContainer = 13
-    kNDArrayHandle = 14
+    kObjectHandle = 7
+    kStr = 8
+    kBytes = 9
+    kPyArg = 10
+    kNDArrayHandle = 11
     kExtBegin = 15
 
 cdef extern from "mxnet/runtime/c_runtime_api.h":
diff --git a/python/mxnet/_ffi/_cython/function.pxi b/python/mxnet/_ffi/_cython/function.pxi
index 2683868cba03..d4c629a618d5 100644
--- a/python/mxnet/_ffi/_cython/function.pxi
+++ b/python/mxnet/_ffi/_cython/function.pxi
@@ -18,6 +18,7 @@
 """Acknowledgement: This file originates from incubator-tvm"""
 
 import ctypes
+import numpy as onp
 import traceback
 from ...ndarray._internal import NDArrayBase
 from numbers import Number, Integral
@@ -58,14 +59,23 @@ cdef inline int make_arg(object arg,
     elif isinstance(arg, ctypes.c_void_p):
         value[0].v_handle = c_handle(arg)
         tcode[0] = kHandle
+    elif isinstance(arg, type):
+        tstr = c_str(onp.dtype(arg).name)
+        value[0].v_str = tstr
+        tcode[0] = kStr
+        temp_args.append(tstr)
     else:
         raise TypeError("Don't know how to handle type %s" % type(arg))
     return 0
 
 
-cdef inline object make_ret(MXNetValue value, int tcode):
+cdef inline object make_ret(MXNetValue value, int tcode, tuple args):
     """convert result to return value."""
-    if tcode == kNull:
+    if tcode == kNDArrayHandle:
+        return c_make_array(value.v_handle)
+    elif tcode == kPyArg:
+        return args[value.v_int64]
+    elif tcode == kNull:
         return None
     elif tcode == kInt:
         return value.v_int64
@@ -75,8 +85,6 @@ cdef inline object make_ret(MXNetValue value, int tcode):
         return py_str(value.v_str)
     elif tcode == kHandle:
         return ctypes_handle(value.v_handle)
-    elif tcode == kNDArrayHandle:
-        return c_make_array(value.v_handle)
     raise ValueError("Unhandled type code %d" % tcode)
 
 
@@ -160,4 +168,4 @@ cdef class FunctionBase:
         cdef MXNetValue ret_val
         cdef int ret_tcode
         FuncCall(self.chandle, args, &ret_val, &ret_tcode)
-        return make_ret(ret_val, ret_tcode)
+        return make_ret(ret_val, ret_tcode, args)
diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 271bb1827b97..279501d385f8 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -134,57 +134,6 @@ def _np_sometrue(a, axis=None, keepdims=False, out=None):
     pass
 
 
-def _np_cumsum(a, axis=None, dtype=None, out=None):
-    """
-    Return the cumulative sum of the elements along a given axis.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array.
-    axis : int, optional
-        Axis along which the cumulative sum is computed. The default
-        (None) is to compute the cumsum over the flattened array.
-    dtype : dtype, optional
-        Type of the returned array and of the accumulator in which the
-        elements are summed.  If `dtype` is not specified, it defaults
-        to the dtype of `a`, unless `a` has an integer dtype with a
-        precision less than that of the default platform integer.  In
-        that case, the default platform integer is used.
-    out : ndarray, optional
-        Alternative output array in which to place the result. It must
-        have the same shape and buffer length as the expected output
-        but the type will be cast if necessary. See `doc.ufuncs`
-        (Section "Output arguments") for more details.
-
-    Returns
-    -------
-    cumsum_along_axis : ndarray.
-        A new array holding the result is returned unless `out` is
-        specified, in which case a reference to `out` is returned. The
-        result has the same size as `a`, and the same shape as `a` if
-        `axis` is not None or `a` is a 1-d array.
-
-    Examples
-    --------
-    >>> a = np.array([[1,2,3], [4,5,6]])
-    >>> a
-    array([[1, 2, 3],
-           [4, 5, 6]])
-    >>> np.cumsum(a)
-    array([ 1,  3,  6, 10, 15, 21])
-    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)
-    array([  1.,   3.,   6.,  10.,  15.,  21.])
-    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns
-    array([[1, 2, 3],
-           [5, 7, 9]])
-    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows
-    array([[ 1,  3,  6],
-           [ 4,  9, 15]])
-    """
-    pass
-
-
 def _npx_nonzero(a):
     """
     Return the indices of the elements that are non-zero.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 82b57fb8cc1f..2d73699afff0 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -46,7 +46,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
-           'where', 'bincount', 'pad']
+           'where', 'bincount', 'pad', 'cumsum']
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -989,7 +989,9 @@ def add(x1, x2, out=None, **kwargs):
         * If only one of the inputs is floating number type, the result is that type.
         * If both inputs are of integer types (including boolean), not supported yet.
     """
-    return _ufunc_helper(x1, x2, _npi.add, _np.add, _npi.add_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        _np.add(x1, x2, out=out)
+    return _api_internal.add(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7637,3 +7639,55 @@ def pad(x, pad_width, mode='constant', **kwargs): # pylint: disable=too-many-arg
             raise ValueError("unsupported stat_length '{}'".format(values))
         return _npi.pad(x, pad_width, mode='minimum')
     return _npi.pad(x, pad_width, mode='constant', constant_value=0)
+
+
+@set_module('mxnet.ndarray.numpy')
+def cumsum(a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative sum of the elements along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis : int, optional
+        Axis along which the cumulative sum is computed. The default
+        (None) is to compute the cumsum over the flattened array.
+    dtype : dtype, optional
+        Type of the returned array and of the accumulator in which the
+        elements are summed.  If `dtype` is not specified, it defaults
+        to the dtype of `a`, unless `a` has an integer dtype with a
+        precision less than that of the default platform integer.  In
+        that case, the default platform integer is used.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type will be cast if necessary. See `doc.ufuncs`
+        (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    cumsum_along_axis : ndarray.
+        A new array holding the result is returned unless `out` is
+        specified, in which case a reference to `out` is returned. The
+        result has the same size as `a`, and the same shape as `a` if
+        `axis` is not None or `a` is a 1-d array.
+
+    Examples
+    --------
+    >>> a = np.array([[1,2,3], [4,5,6]])
+    >>> a
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> np.cumsum(a)
+    array([ 1,  3,  6, 10, 15, 21])
+    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)
+    array([  1.,   3.,   6.,  10.,  15.,  21.])
+    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns
+    array([[1, 2, 3],
+           [5, 7, 9]])
+    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows
+    array([[ 1,  3,  6],
+           [ 4,  9, 15]])
+    """
+    return _api_internal.cumsum(a, axis, dtype, out)
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 3ff984c7a2a3..12184a651c3d 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -66,7 +66,8 @@
            'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal',
            'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
-           'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount', 'pad']
+           'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
+           'pad', 'cumsum']
 
 __all__ += fallback.__all__
 
@@ -1795,7 +1796,7 @@ def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):
 
     def cumsum(self, axis=None, dtype=None, out=None):
         """Return the cumulative sum of the elements along the given axis."""
-        return _mx_np_op.cumsum(self, axis=axis, dtype=dtype, out=out)
+        return _mx_nd_np.cumsum(self, axis=axis, dtype=dtype, out=out)
 
     def tolist(self):
         return self.asnumpy().tolist()
@@ -9685,3 +9686,55 @@ def pad(x, pad_width=None, mode="constant", **kwargs): # pylint: disable=too-man
            [10, 10, 10, 10, 10, 10, 10]])
     """
     return _mx_nd_np.pad(x, pad_width, mode, **kwargs)
+
+
+@set_module('mxnet.numpy')
+def cumsum(a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative sum of the elements along a given axis.
+
+    Parameters
+    ----------
+    a : array_like
+        Input array.
+    axis : int, optional
+        Axis along which the cumulative sum is computed. The default
+        (None) is to compute the cumsum over the flattened array.
+    dtype : dtype, optional
+        Type of the returned array and of the accumulator in which the
+        elements are summed.  If `dtype` is not specified, it defaults
+        to the dtype of `a`, unless `a` has an integer dtype with a
+        precision less than that of the default platform integer.  In
+        that case, the default platform integer is used.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type will be cast if necessary. See `doc.ufuncs`
+        (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    cumsum_along_axis : ndarray.
+        A new array holding the result is returned unless `out` is
+        specified, in which case a reference to `out` is returned. The
+        result has the same size as `a`, and the same shape as `a` if
+        `axis` is not None or `a` is a 1-d array.
+
+    Examples
+    --------
+    >>> a = np.array([[1,2,3], [4,5,6]])
+    >>> a
+    array([[1, 2, 3],
+           [4, 5, 6]])
+    >>> np.cumsum(a)
+    array([ 1,  3,  6, 10, 15, 21])
+    >>> np.cumsum(a, dtype=float)     # specifies type of output value(s)
+    array([  1.,   3.,   6.,  10.,  15.,  21.])
+    >>> np.cumsum(a,axis=0)      # sum over rows for each of the 3 columns
+    array([[1, 2, 3],
+           [5, 7, 9]])
+    >>> np.cumsum(a,axis=1)      # sum over columns for each of the 2 rows
+    array([[ 1,  3,  6],
+           [ 4,  9, 15]])
+    """
+    return _mx_nd_np.cumsum(a, axis=axis, dtype=dtype, out=out)
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 8756b6a78ac9..bf3e50ba1388 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -51,7 +51,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
-           'where', 'bincount', 'pad']
+           'where', 'bincount', 'pad', 'cumsum']
 
 
 @set_module('mxnet.symbol.numpy')
@@ -683,7 +683,7 @@ def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylin
 
     def cumsum(self, axis=None, dtype=None, out=None):
         """Return the cumulative sum of the elements along the given axis."""
-        return _mx_np_op.cumsum(self, axis=axis, dtype=dtype, out=out)
+        return _npi.cumsum(self, axis=axis, dtype=dtype, out=out)
 
     def max(self, axis=None, out=None, keepdims=False):  # pylint: disable=arguments-differ
         """Return the maximum along a given axis."""
@@ -6732,4 +6732,39 @@ def pad(x, pad_width, mode='constant', **kwargs): # pylint: disable=too-many-arg
     return _npi.pad(x, pad_width, mode='constant', constant_value=0)
 
 
+@set_module('mxnet.symbol.numpy')
+def cumsum(a, axis=None, dtype=None, out=None):
+    """
+    Return the cumulative sum of the elements along a given axis.
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input array.
+    axis : int, optional
+        Axis along which the cumulative sum is computed. The default
+        (None) is to compute the cumsum over the flattened array.
+    dtype : dtype, optional
+        Type of the returned array and of the accumulator in which the
+        elements are summed.  If `dtype` is not specified, it defaults
+        to the dtype of `a`, unless `a` has an integer dtype with a
+        precision less than that of the default platform integer.  In
+        that case, the default platform integer is used.
+    out : _Symbol, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output
+        but the type will be cast if necessary. See `doc.ufuncs`
+        (Section "Output arguments") for more details.
+
+    Returns
+    -------
+    cumsum_along_axis : _Symbol.
+        A new array holding the result is returned unless `out` is
+        specified, in which case a reference to `out` is returned. The
+        result has the same size as `a`, and the same shape as `a` if
+        `axis` is not None or `a` is a 1-d array.
+    """
+    return _npi.cumsum(a, axis=axis, dtype=dtype, out=out)
+
+
 _set_np_symbol_class(_Symbol)
diff --git a/src/api/operator/numpy/np_cumsum.cc b/src/api/operator/numpy/np_cumsum.cc
new file mode 100644
index 000000000000..0ef3b3fdf7bf
--- /dev/null
+++ b/src/api/operator/numpy/np_cumsum.cc
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_cumsum.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_cumsum.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_cumsum-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.cumsum")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = Op::Get("_npi_cumsum");
+  op::CumsumParam param;
+  // axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[1].operator int();
+  }
+  // dtype
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::nullopt;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::CumsumParam>(&attrs);
+  // inputs
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+  int num_inputs = 1;
+  // outputs
+  NDArray* outputs[] = {args[3].operator NDArray*()};
+  NDArray** out = outputs[0] == nullptr ? nullptr : outputs;
+  int num_outputs = outputs[0] != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_elemwise_broadcast_op.cc b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
new file mode 100644
index 000000000000..e724a7c58bd3
--- /dev/null
+++ b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_elemwise_broadcast_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_elemwise_broadcast_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../ufunc_helper.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.add")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_add");
+  const nnvm::Op* op_scalar = Op::Get("_npi_add_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_init_op.cc b/src/api/operator/numpy/np_init_op.cc
index 746985c6e9f3..c65f90c841f4 100644
--- a/src/api/operator/numpy/np_init_op.cc
+++ b/src/api/operator/numpy/np_init_op.cc
@@ -21,6 +21,8 @@
  * \file np_init_op.cc
  * \brief Implementation of the API of functions in src/operator/numpy/np_init_op.cc
  */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
 #include "../../../operator/tensor/init_op.h"
 
@@ -44,11 +46,12 @@ MXNET_REGISTER_API("_npi.zeros")
   }
   attrs.parsed = std::move(param);
   attrs.op = op;
+  SetAttrDict<op::InitOpParam>(&attrs);
   if (args[2].type_code() != kNull) {
     attrs.dict["ctx"] = args[2].operator std::string();
   }
   int num_outputs = 0;
-  auto ndoutputs = Invoke<op::InitOpParam>(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
   *ret = ndoutputs[0];
 });
 
diff --git a/src/api/operator/numpy/np_tensordot_op.cc b/src/api/operator/numpy/np_tensordot_op.cc
index ade2a0314d01..b163757f85b1 100644
--- a/src/api/operator/numpy/np_tensordot_op.cc
+++ b/src/api/operator/numpy/np_tensordot_op.cc
@@ -21,6 +21,7 @@
  * \file np_tensordot_op.cc
  * \brief Implementation of the API of functions in src/operator/numpy/np_tensordot_op.cc
  */
+#include <mxnet/api_registry.h>
 #include "../utils.h"
 #include "../../../operator/numpy/np_tensordot_op-inl.h"
 
@@ -32,13 +33,14 @@ inline static void _npi_tensordot_int_axes(runtime::MXNetArgs args,
   const nnvm::Op* op = Op::Get("_npi_tensordot_int_axes");
   op::TensordotIntAxesParam param;
   nnvm::NodeAttrs attrs;
-  attrs.op = op;
   param.axes = args[2].operator int();
+  attrs.op = op;
   // we directly copy TensordotIntAxesParam, which is trivially-copyable
   attrs.parsed = param;
+  SetAttrDict<op::TensordotIntAxesParam>(&attrs);
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke<op::TensordotIntAxesParam>(op, &attrs, 2, inputs, &num_outputs, nullptr);
+  auto ndoutputs = Invoke(op, &attrs, 2, inputs, &num_outputs, nullptr);
   *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
 }
 
@@ -48,7 +50,6 @@ inline static void _npi_tensordot(runtime::MXNetArgs args,
   const nnvm::Op* op = Op::Get("_npi_tensordot");
   op::TensordotParam param;
   nnvm::NodeAttrs attrs;
-  attrs.op = op;
   ADT adt = Downcast<ADT, ObjectRef>(args[2].operator ObjectRef());
   if (const IntegerObj* lop = adt[0].as<IntegerObj>()) {
     param.a_axes_summed = Tuple<int>(1, lop->value);
@@ -57,10 +58,12 @@ inline static void _npi_tensordot(runtime::MXNetArgs args,
     param.a_axes_summed = Tuple<int>(adt[0]);
     param.b_axes_summed = Tuple<int>(adt[1]);
   }
+  attrs.op = op;
   attrs.parsed = std::move(param);
+  SetAttrDict<op::TensordotParam>(&attrs);
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke<op::TensordotParam>(op, &attrs, 2, inputs, &num_outputs, nullptr);
+  auto ndoutputs = Invoke(op, &attrs, 2, inputs, &num_outputs, nullptr);
   *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
 }
 
diff --git a/src/api/operator/op_utils.cc b/src/api/operator/op_utils.cc
new file mode 100644
index 000000000000..220a880336db
--- /dev/null
+++ b/src/api/operator/op_utils.cc
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file op_utils.cc
+ * \brief Utility functions for modification in src/operator
+ */
+
+#include "op_utils.h"
+#include <mxnet/base.h>
+
+namespace mxnet {
+
+std::string String2MXNetTypeWithBool(int dtype) {
+  switch (dtype) {
+    case mshadow::kFloat32:
+      return "float32";
+    case mshadow::kFloat64:
+      return "float64";
+    case mshadow::kFloat16:
+      return "float16";
+    case mshadow::kUint8:
+      return "uint8";
+    case mshadow::kInt8:
+      return "int8";
+    case mshadow::kInt32:
+      return "int32";
+    case mshadow::kInt64:
+      return "int64";
+    case mshadow::kBool:
+      return "bool";
+    default:
+      LOG(FATAL) << "Unknown type enum " << dtype;
+  }
+  LOG(FATAL) << "should not reach here ";
+  return "";
+}
+
+}  // namespace mxnet
diff --git a/src/api/operator/op_utils.h b/src/api/operator/op_utils.h
new file mode 100644
index 000000000000..4c577983c405
--- /dev/null
+++ b/src/api/operator/op_utils.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file op_utils.h
+ * \brief Utility functions for modification in src/operator
+ */
+#ifndef MXNET_API_OPERATOR_OP_UTILS_H_
+#define MXNET_API_OPERATOR_OP_UTILS_H_
+
+#include <string>
+
+namespace mxnet {
+
+std::string String2MXNetTypeWithBool(int dtype);
+
+}  // namespace mxnet
+
+#endif  // MXNET_API_OPERATOR_OP_UTILS_H_
diff --git a/src/api/operator/ufunc_helper.cc b/src/api/operator/ufunc_helper.cc
new file mode 100644
index 000000000000..67bc68031417
--- /dev/null
+++ b/src/api/operator/ufunc_helper.cc
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ufunc_helper.cc
+ * \brief ufunc helper
+ */
+#include "ufunc_helper.h"
+#include "utils.h"
+
+namespace mxnet {
+
+template<>
+void SetAttrDict<double>(nnvm::NodeAttrs* attrs) {
+  if (Imperative::Get()->is_recording()) {
+    attrs->dict["scalar"] = std::to_string(::dmlc::get<double>(attrs->parsed));
+  }
+}
+
+void UFuncHelper(NDArray* lhs, NDArray* rhs, NDArray* out,
+                 runtime::MXNetRetValue* ret, const nnvm::Op* op) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  NDArray* inputs[] = {lhs, rhs};
+  int num_inputs = 2;
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (outputs) {
+    *ret = PythonArg(2);
+  } else {
+    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);
+  }
+}
+
+void UFuncHelper(NDArray* lhs, double rhs, NDArray* out,
+                 runtime::MXNetRetValue* ret, const nnvm::Op* op) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  attrs.parsed = rhs;
+  SetAttrDict<double>(&attrs);
+  NDArray** inputs = &lhs;
+  int num_inputs = 1;
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (outputs) {
+    *ret = PythonArg(2);
+  } else {
+    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);
+  }
+}
+
+void UFuncHelper(double lhs, NDArray* rhs, NDArray* out,
+                 runtime::MXNetRetValue* ret, const nnvm::Op* op) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  attrs.parsed = lhs;
+  SetAttrDict<double>(&attrs);
+  NDArray** inputs = &rhs;
+  int num_inputs = 1;
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (outputs) {
+    *ret = PythonArg(2);
+  } else {
+    *ret = reinterpret_cast<NDArray*>(ndoutputs[0]);
+  }
+}
+
+void UFuncHelper(runtime::MXNetArgs args,
+                 runtime::MXNetRetValue* ret,
+                 const nnvm::Op* fn_array,
+                 const nnvm::Op* lfn_scalar,
+                 const nnvm::Op* rfn_scalar) {
+  using namespace runtime;
+  NDArray* out = args[2].operator NDArray*();
+  if (args[0].type_code() == kNDArrayHandle) {
+    if (args[1].type_code() == kNDArrayHandle) {
+      UFuncHelper(args[0].operator NDArray*(), args[1].operator NDArray*(), out, ret, fn_array);
+    } else {
+      UFuncHelper(args[0].operator NDArray*(), args[1].operator double(), out, ret, lfn_scalar);
+    }
+  } else {
+    UFuncHelper(args[0].operator double(), args[1].operator NDArray*(), out, ret,
+                rfn_scalar ? rfn_scalar : lfn_scalar);
+  }
+}
+
+}  // namespace mxnet
diff --git a/src/api/operator/ufunc_helper.h b/src/api/operator/ufunc_helper.h
new file mode 100644
index 000000000000..793d0b22ed9f
--- /dev/null
+++ b/src/api/operator/ufunc_helper.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ufunc_helper.h
+ * \brief ufunc helper
+ */
+#ifndef MXNET_API_OPERATOR_UFUNC_HELPER_H_
+#define MXNET_API_OPERATOR_UFUNC_HELPER_H_
+#include <mxnet/runtime/packed_func.h>
+namespace mxnet {
+
+void UFuncHelper(runtime::MXNetArgs args,
+                 runtime::MXNetRetValue* ret,
+                 const nnvm::Op* fn_array,
+                 const nnvm::Op* lfn_scalar,
+                 const nnvm::Op* rfn_scalar);
+}  // namespace mxnet
+
+#endif  // MXNET_API_OPERATOR_UFUNC_HELPER_H_
diff --git a/src/api/operator/utils.cc b/src/api/operator/utils.cc
index d8cd4c922603..3d8401270a40 100644
--- a/src/api/operator/utils.cc
+++ b/src/api/operator/utils.cc
@@ -66,4 +66,26 @@ void SetInOut(std::vector<NDArray*>* ndinputs,
   }
 }
 
+std::vector<NDArray*> Invoke(const nnvm::Op* op,
+                             nnvm::NodeAttrs* attrs,
+                             int num_inputs,
+                             NDArray** inputs,
+                             int* num_outputs,
+                             NDArray** outputs) {
+  int infered_num_outputs;
+  int num_visible_outputs;
+  imperative::SetNumOutputs(op, *attrs, num_inputs, &infered_num_outputs, &num_visible_outputs);
+
+  std::vector<NDArray*> ndinputs, ndoutputs;
+  SetInOut(&ndinputs, &ndoutputs, num_inputs, inputs,
+      num_outputs, infered_num_outputs, num_visible_outputs, outputs);
+
+  auto state = Imperative::Get()->Invoke(Context::CPU(), *attrs, ndinputs, ndoutputs);
+  if (Imperative::Get()->is_recording()) {
+    Imperative::Get()->RecordOp(std::move(*attrs), ndinputs, ndoutputs, state);
+  }
+  for (int i = *num_outputs; i < infered_num_outputs; ++i) delete ndoutputs[i];
+  return ndoutputs;
+}
+
 }  // namespace mxnet
diff --git a/src/api/operator/utils.h b/src/api/operator/utils.h
index 7a31e4537780..49ee6bf2c9af 100644
--- a/src/api/operator/utils.h
+++ b/src/api/operator/utils.h
@@ -24,13 +24,10 @@
 #ifndef MXNET_API_OPERATOR_UTILS_H_
 #define MXNET_API_OPERATOR_UTILS_H_
 
-#include <mxnet/runtime/ffi_helper.h>
-#include <mxnet/runtime/container.h>
-#include <mxnet/runtime/packed_func.h>
-#include <mxnet/api_registry.h>
 #include <mxnet/base.h>
 #include <nnvm/c_api.h>
 #include <vector>
+#include <string>
 #include "../../imperative/imperative_utils.h"
 
 namespace mxnet {
@@ -44,28 +41,18 @@ void SetInOut(std::vector<NDArray*>* ndinputs,
               int num_visible_outputs,
               NDArray** out_array);
 
-template<typename T>
 std::vector<NDArray*> Invoke(const nnvm::Op* op,
                              nnvm::NodeAttrs* attrs,
                              int num_inputs,
                              NDArray** inputs,
                              int* num_outputs,
-                             NDArray** outputs) {
-  int infered_num_outputs;
-  int num_visible_outputs;
-  imperative::SetNumOutputs(op, *attrs, num_inputs, &infered_num_outputs, &num_visible_outputs);
-
-  std::vector<NDArray*> ndinputs, ndoutputs;
-  SetInOut(&ndinputs, &ndoutputs, num_inputs, inputs,
-      num_outputs, infered_num_outputs, num_visible_outputs, outputs);
+                             NDArray** outputs);
 
-  auto state = Imperative::Get()->Invoke(Context::CPU(), *attrs, ndinputs, ndoutputs);
+template<typename T>
+void SetAttrDict(nnvm::NodeAttrs* attrs) {
   if (Imperative::Get()->is_recording()) {
     ::dmlc::get<T>(attrs->parsed).SetAttrDict(&(attrs->dict));
-    Imperative::Get()->RecordOp(std::move(*attrs), ndinputs, ndoutputs, state);
   }
-  for (int i = *num_outputs; i < infered_num_outputs; ++i) delete ndoutputs[i];
-  return ndoutputs;
 }
 
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_cumsum-inl.h b/src/operator/numpy/np_cumsum-inl.h
index 375d83b2240f..dfba843d7be6 100644
--- a/src/operator/numpy/np_cumsum-inl.h
+++ b/src/operator/numpy/np_cumsum-inl.h
@@ -28,9 +28,11 @@
 #include <mxnet/base.h>
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../mxnet_op.h"
 #include "../operator_common.h"
 #include "../elemwise_op_common.h"
+#include "../../api/operator/op_utils.h"
 
 namespace mxnet {
 namespace op {
@@ -56,6 +58,17 @@ struct CumsumParam : public dmlc::Parameter<CumsumParam> {
                 " unless a has an integer dtype with a precision less than that of the"
                 " default platform integer. In that case, the default platform integer is used.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, dtype_s;
+    axis_s << axis;
+    dtype_s << dtype;
+    (*dict)["axis"] = axis_s.str();
+    if (dtype.has_value()) {
+      (*dict)["dtype"] = String2MXNetTypeWithBool(dtype.value());
+    } else {
+      (*dict)["dtype"] = dtype_s.str();
+    }
+  }
 };
 
 struct cumsum_forward {
diff --git a/src/operator/numpy/np_cumsum.cc b/src/operator/numpy/np_cumsum.cc
index 2d5dbb99f90a..ea0f9b6b11bc 100644
--- a/src/operator/numpy/np_cumsum.cc
+++ b/src/operator/numpy/np_cumsum.cc
@@ -65,8 +65,7 @@ inline bool CumsumType(const nnvm::NodeAttrs& attrs,
 
 DMLC_REGISTER_PARAMETER(CumsumParam);
 
-NNVM_REGISTER_OP(_np_cumsum)
-.add_alias("cumsum")
+NNVM_REGISTER_OP(_npi_cumsum)
 .describe(R"code(Return the cumulative sum of the elements along a given axis.)code" ADD_FILELINE)
 .set_attr_parser(ParamParser<CumsumParam>)
 .set_num_inputs(1)
@@ -78,7 +77,7 @@ NNVM_REGISTER_OP(_np_cumsum)
 .set_attr<mxnet::FInferShape>("FInferShape", CumsumShape)
 .set_attr<nnvm::FInferType>("FInferType", CumsumType)
 .set_attr<FCompute>("FCompute<cpu>", CumsumForward<cpu>)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_np_cumsum"})
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_npi_cumsum"})
 .set_attr<nnvm::FInplaceOption>("FInplaceOption",
   [](const NodeAttrs& attrs) {
     return std::vector<std::pair<int, int> >{{0, 0}};
@@ -86,7 +85,7 @@ NNVM_REGISTER_OP(_np_cumsum)
 .add_argument("a", "NDArray-or-Symbol", "Input ndarray")
 .add_arguments(CumsumParam::__FIELDS__());
 
-NNVM_REGISTER_OP(_backward_np_cumsum)
+NNVM_REGISTER_OP(_backward_npi_cumsum)
 .set_attr_parser(ParamParser<CumsumParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
diff --git a/src/operator/numpy/np_cumsum.cu b/src/operator/numpy/np_cumsum.cu
index cc574ebf72c5..438bab2b5efe 100644
--- a/src/operator/numpy/np_cumsum.cu
+++ b/src/operator/numpy/np_cumsum.cu
@@ -27,10 +27,10 @@
 namespace mxnet {
 namespace op {
 
-NNVM_REGISTER_OP(_np_cumsum)
+NNVM_REGISTER_OP(_npi_cumsum)
 .set_attr<FCompute>("FCompute<gpu>", CumsumForward<gpu>);
 
-NNVM_REGISTER_OP(_backward_np_cumsum)
+NNVM_REGISTER_OP(_backward_npi_cumsum)
 .set_attr<FCompute>("FCompute<gpu>", CumsumBackward<gpu>);
 
 }  // namespace op

From e8e320ec5981d81d7692f7de8b8f3a9fe272aac7 Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Wed, 11 Mar 2020 06:02:00 +0800
Subject: [PATCH 04/44] [Numpy] FFI: Bincount, Percentile/Quantile, All/Any
 (#17717)

* ffi_bincount percentile/quantile all/any

* new ffi
---
 python/mxnet/_numpy_op_doc.py                 | 101 --------------
 python/mxnet/ndarray/numpy/_op.py             | 124 +++++++++++++++---
 python/mxnet/numpy/multiarray.py              | 116 +++++++++++++++-
 python/mxnet/symbol/numpy/_symbol.py          |  63 ++++++++-
 src/api/operator/numpy/np_bincount_op.cc      |  61 +++++++++
 .../numpy/np_broadcast_reduce_op_boolean.cc   |  93 +++++++++++++
 src/api/operator/numpy/np_percentile_op.cc    |  98 ++++++++++++++
 src/api/operator/op_utils.cc                  |  21 +++
 src/api/operator/op_utils.h                   |   1 +
 src/operator/numpy/np_bincount_op-inl.h       |   8 ++
 src/operator/numpy/np_broadcast_reduce_op.h   |   7 +
 .../numpy/np_broadcast_reduce_op_boolean.cc   |   4 +-
 .../numpy/np_broadcast_reduce_op_boolean.cu   |   4 +-
 src/operator/numpy/np_percentile_op-inl.h     |  12 ++
 tests/python/unittest/test_numpy_op.py        |   2 -
 15 files changed, 586 insertions(+), 129 deletions(-)
 create mode 100644 src/api/operator/numpy/np_bincount_op.cc
 create mode 100644 src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc
 create mode 100644 src/api/operator/numpy/np_percentile_op.cc

diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 279501d385f8..8dfc0867cdb2 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -20,107 +20,6 @@
 """Doc placeholder for numpy ops with prefix _np."""
 
 
-def _np_all(a, axis=None, keepdims=False, out=None):
-    """
-    Test whether all array elements along a given axis evaluate to True.
-
-    Parameters
-    ----------
-    a : array_like
-        Input array or object that can be converted to an array.
-    axis : None or int or tuple of ints, optional
-        Axis or axes along which a logical AND reduction is performed.
-        The default (axis = None) is to perform a logical AND over
-        all the dimensions of the input array.
-    keepdims : bool, optional
-        If this is set to True, the axes which are reduced are left in
-        the result as dimensions with size one. With this option,
-        the result will broadcast correctly against the input array.
-    out : ndarray, optional
-        Alternate output array in which to place the result. It must have
-        the same shape as the expected output and its type is preserved
-
-    Returns
-    --------
-    all : ndarray, bool
-        A new boolean or array is returned unless out is specified,
-        in which case a reference to out is returned.
-
-    Examples:
-    ---------
-    >>> np.all([[True,False],[True,True]])
-    False
-
-    >>> np.all([[True,False],[True,True]], axis=0)
-    array([ True, False])
-
-    >>> np.all([-1, 4, 5])
-    True
-
-    >>> np.all([1.0, np.nan])
-    True
-
-    >>> o=np.array(False)
-    >>> z=np.all([-1, 4, 5], out=o)
-    >>> id(z), id(o), z
-    (28293632, 28293632, array(True)) # may vary
-    """
-    pass
-
-def _np_any(a, axis=None, keepdims=False, out=None):
-    """
-    Test whether any array element along a given axis evaluates to True.
-    Returns single boolean unless axis is not None 
-
-    Parameters
-    ----------
-    a : array_like
-        Input array or object that can be converted to an array.
-    axis : None or int or tuple of ints, optional
-        Axis or axes along which a logical AND reduction is performed.
-        The default (axis = None) is to perform a logical AND over
-        all the dimensions of the input array.
-    keepdims : bool, optional
-        If this is set to True, the axes which are reduced are left in
-        the result as dimensions with size one. With this option,
-        the result will broadcast correctly against the input array.
-    out : ndarray, optional
-        Alternate output array in which to place the result. It must have
-        the same shape as the expected output and its type is preserved
-
-    Returns
-    --------
-    any : bool or ndarray
-        A new boolean or ndarray is returned unless out is specified,
-        in which case a reference to out is returned.
-
-    Examples:
-    ---------
-    >>> np.any([[True, False], [True, True]])
-    True
-
-    >>> np.any([[True, False], [False, False]], axis=0)
-    array([ True, False])
-
-    >>> np.any([-1, 0, 5])
-    True
-
-    >>> np.any(np.nan)
-    True
-
-    >>> o=np.array(False)
-    >>> z=np.any([-1, 4, 5], out=o)
-    >>> z, o
-    (array(True), array(True))
-    >>> # Check now that z is a reference to o
-    >>> z is o
-    True
-    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP
-    (191614240, 191614240)
-    """
-    pass
-
-
 def _np_sometrue(a, axis=None, keepdims=False, out=None):
     """
     Check whether some values are true.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 2d73699afff0..b29586fca190 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -34,7 +34,7 @@
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'insert', 'fabs',
            'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul',
            'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram',
-           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort',
+           'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'all', 'any', 'sort',
            'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
@@ -1380,6 +1380,110 @@ def power(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.power, _np.power, _npi.power_scalar, _npi.rpower_scalar, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+def all(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether all array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    all : ndarray, bool
+        A new boolean or array is returned unless out is specified,
+        in which case a reference to out is returned.
+
+    Examples:
+    ---------
+    >>> np.all([[True,False],[True,True]])
+    False
+
+    >>> np.all([[True,False],[True,True]], axis=0)
+    array([ True, False])
+
+    >>> np.all([-1, 4, 5])
+    True
+
+    >>> np.all([1.0, np.nan])
+    True
+
+    >>> o=np.array(False)
+    >>> z=np.all([-1, 4, 5], out=o)
+    >>> id(z), id(o), z
+    (28293632, 28293632, array(True)) # may vary
+    """
+    return _api_internal.all(a, axis, keepdims, out)
+
+
+@set_module('mxnet.ndarray.numpy')
+def any(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether any array element along a given axis evaluates to True.
+    Returns single boolean unless axis is not None
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    any : bool or ndarray
+        A new boolean or ndarray is returned unless out is specified,
+        in which case a reference to out is returned.
+
+    Examples:
+    ---------
+    >>> np.any([[True, False], [True, True]])
+    True
+
+    >>> np.any([[True, False], [False, False]], axis=0)
+    array([ True, False])
+
+    >>> np.any([-1, 0, 5])
+    True
+
+    >>> np.any(np.nan)
+    True
+
+    >>> o=np.array(False)
+    >>> z=np.any([-1, 4, 5], out=o)
+    >>> z, o
+    (array(True), array(True))
+    >>> # Check now that z is a reference to o
+    >>> z is o
+    True
+    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP
+    (191614240, 191614240)
+    """
+    return _api_internal.any(a, axis, keepdims, out)
+
+
 @set_module('mxnet.ndarray.numpy')
 def argsort(a, axis=-1, kind=None, order=None):
     """
@@ -6637,11 +6741,7 @@ def percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='l
     """
     if overwrite_input is not None:
         raise NotImplementedError('overwrite_input is not supported yet')
-    if isinstance(q, numeric_types):
-        return _npi.percentile(a, axis=axis, interpolation=interpolation,
-                               keepdims=keepdims, q_scalar=q, out=out)
-    return _npi.percentile(a, q, axis=axis, interpolation=interpolation,
-                           keepdims=keepdims, q_scalar=None, out=out)
+    return _api_internal.percentile(a, q, axis, interpolation, keepdims, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6722,11 +6822,7 @@ def quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='lin
     """
     if overwrite_input is not None:
         raise NotImplementedError('overwrite_input is not supported yet')
-    if isinstance(q, numeric_types):
-        return _npi.percentile(a, axis=axis, interpolation=interpolation,
-                               keepdims=keepdims, q_scalar=q * 100, out=out)
-    return _npi.percentile(a, q * 100, axis=axis, interpolation=interpolation,
-                           keepdims=keepdims, q_scalar=None, out=out)
+    return _api_internal.percentile(a, q * 100, axis, interpolation, keepdims, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7498,13 +7594,9 @@ def bincount(x, weights=None, minlength=0):
     >>> np.bincount(x,  weights=w)
     array([ 0.3,  0.7,  1.1])
     """
-    if not isinstance(x, NDArray):
-        raise TypeError("Input data should be NDarray")
     if minlength < 0:
         raise ValueError("Minlength value should greater than 0")
-    if weights is None:
-        return _npi.bincount(x, minlength=minlength, has_weights=False)
-    return _npi.bincount(x, weights=weights, minlength=minlength, has_weights=True)
+    return _api_internal.bincount(x, weights, minlength)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 12184a651c3d..a5024253c162 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -22,8 +22,10 @@
 
 
 try:
+    from __builtin__ import all as py_all
     from __builtin__ import slice as py_slice
 except ImportError:
+    from builtins import all as py_all
     from builtins import slice as py_slice
 
 from array import array as native_array
@@ -50,7 +52,7 @@
 
 
 __all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape',
-           'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'broadcast_to',
+           'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'all', 'any', 'broadcast_to',
            'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'bitwise_not', 'delete',
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'invert',
            'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log',
@@ -297,7 +299,7 @@ def __array_function__(self, func, types, args, kwargs):  # pylint: disable=bad-
         else:
             # Note: this allows subclasses that don't override
             # __array_function__ to handle mxnet.numpy.ndarray objects
-            if not all(issubclass(t, ndarray) for t in types):
+            if not py_all(issubclass(t, ndarray) for t in types):
                 return NotImplemented
             return mx_np_func(*args, **kwargs)
 
@@ -613,7 +615,7 @@ def __getitem__(self, key):
         if isinstance(key, tuple) and len(key) == 0:
             return self
         if isinstance(key, tuple) and len(key) == ndim\
-                and all(isinstance(idx, integer_types) for idx in key):
+                and py_all(isinstance(idx, integer_types) for idx in key):
             out = self
             for idx in key:
                 out = out[idx]
@@ -1078,10 +1080,10 @@ def T(self):
     # pylint: enable= invalid-name, undefined-variable
 
     def all(self, axis=None, out=None, keepdims=False):
-        return _mx_nd_np.all(self, axis=axis, keepdims=keepdims, out=out)
+        return _mx_nd_np.all(self, axis=axis, out=out, keepdims=keepdims)
 
     def any(self, axis=None, out=None, keepdims=False):
-        return _mx_nd_np.any(self, axis=axis, keepdims=keepdims, out=out)
+        return _mx_nd_np.any(self, axis=axis, out=out, keepdims=keepdims)
 
     def as_nd_ndarray(self):
         """Convert mxnet.numpy.ndarray to mxnet.ndarray.NDArray to use its fluent methods."""
@@ -2554,6 +2556,110 @@ def empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pyl
     return _mx_nd_np.empty_like(prototype, dtype=dtype, order=order, subok=subok, shape=shape)
 
 
+@set_module('mxnet.numpy')
+def all(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether all array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    all : ndarray, bool
+        A new boolean or array is returned unless out is specified,
+        in which case a reference to out is returned.
+
+    Examples:
+    ---------
+    >>> np.all([[True,False],[True,True]])
+    False
+
+    >>> np.all([[True,False],[True,True]], axis=0)
+    array([ True, False])
+
+    >>> np.all([-1, 4, 5])
+    True
+
+    >>> np.all([1.0, np.nan])
+    True
+
+    >>> o=np.array(False)
+    >>> z=np.all([-1, 4, 5], out=o)
+    >>> id(z), id(o), z
+    (28293632, 28293632, array(True)) # may vary
+    """
+    return _mx_nd_np.all(a, axis=axis, out=out, keepdims=keepdims)
+
+
+@set_module('mxnet.numpy')
+def any(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether any array element along a given axis evaluates to True.
+    Returns single boolean unless axis is not None
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    any : bool or ndarray
+        A new boolean or ndarray is returned unless out is specified,
+        in which case a reference to out is returned.
+
+    Examples:
+    ---------
+    >>> np.any([[True, False], [True, True]])
+    True
+
+    >>> np.any([[True, False], [False, False]], axis=0)
+    array([ True, False])
+
+    >>> np.any([-1, 0, 5])
+    True
+
+    >>> np.any(np.nan)
+    True
+
+    >>> o=np.array(False)
+    >>> z=np.any([-1, 4, 5], out=o)
+    >>> z, o
+    (array(True), array(True))
+    >>> # Check now that z is a reference to o
+    >>> z is o
+    True
+    >>> id(z), id(o) # identity of z and o              # doctest: +SKIP
+    (191614240, 191614240)
+    """
+    return _mx_nd_np.any(a, axis=axis, out=out, keepdims=keepdims)
+
+
 @set_module('mxnet.numpy')
 def identity(n, dtype=None, ctx=None):
     """
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index bf3e50ba1388..cdfa447c0ebd 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -43,7 +43,7 @@
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero',
+           'average', 'mean', 'maximum', 'minimum', 'any', 'all', 'around', 'round', 'round_', 'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
@@ -4094,6 +4094,67 @@ def minimum(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.minimum, _np.minimum, _npi.minimum_scalar, None, out)
 
 
+@set_module('mxnet.symbol.numpy')
+def all(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether all array elements along a given axis evaluate to True.
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    all : _Symbol, bool
+        A new boolean or array is returned unless out is specified,
+        in which case a reference to out is returned.
+    """
+    return _npi.all(a, axis=axis, keepdims=keepdims, out=out)
+
+
+@set_module('mxnet.symbol.numpy')
+def any(a, axis=None, out=None, keepdims=False):
+    """
+    Test whether any array element along a given axis evaluates to True.
+    Returns single boolean unless axis is not None
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input array or object that can be converted to an array.
+    axis : None or int or tuple of ints, optional
+        Axis or axes along which a logical AND reduction is performed.
+        The default (axis = None) is to perform a logical AND over
+        all the dimensions of the input array.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left in
+        the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+    out : ndarray, optional
+        Alternate output array in which to place the result. It must have
+        the same shape as the expected output and its type is preserved
+
+    Returns
+    --------
+    any : bool or _Symbol
+        A new boolean or ndarray is returned unless out is specified,
+        in which case a reference to out is returned.
+    """
+    return _npi.any(a, axis=axis, keepdims=keepdims, out=out)
+
+
 @set_module('mxnet.symbol.numpy')
 def clip(a, a_min, a_max, out=None):
     """clip(a, a_min, a_max, out=None)
diff --git a/src/api/operator/numpy/np_bincount_op.cc b/src/api/operator/numpy/np_bincount_op.cc
new file mode 100644
index 000000000000..afa3278c24e4
--- /dev/null
+++ b/src/api/operator/numpy/np_bincount_op.cc
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_bincount_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_bincount_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_bincount_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.bincount")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_bincount");
+  nnvm::NodeAttrs attrs;
+  op::NumpyBincountParam param;
+
+  int num_outputs = 0;
+  if (args[1].type_code() == kNull) {
+    param.minlength = args[2].operator int64_t();
+    param.has_weights = false;
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+    int num_inputs = 1;
+    attrs.parsed = param;
+    attrs.op = op;
+    SetAttrDict<op::NumpyBincountParam>(&attrs);
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  } else {
+    param.minlength = args[2].operator int64_t();
+    param.has_weights = true;
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+    int num_inputs = 2;
+    attrs.parsed = param;
+    attrs.op = op;
+    SetAttrDict<op::NumpyBincountParam>(&attrs);
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+    }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc b/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc
new file mode 100644
index 000000000000..dea510a41608
--- /dev/null
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_boolean.cc
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_broadcast_reduce_op_boolean.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_broadcast_reduce_op_boolean.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_broadcast_reduce_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.all")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_all");
+  nnvm::NodeAttrs attrs;
+  op::NumpyReduceAxesBoolParam param;
+
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else if (args[1].type_code() == kDLInt) {
+    param.axis = Tuple<int>(1, args[1].operator int64_t());
+  } else {
+    param.axis = Tuple<int>(args[1].operator ObjectRef());
+  }
+  param.keepdims = args[2].operator bool();
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyReduceAxesBoolParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+MXNET_REGISTER_API("_npi.any")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_any");
+  nnvm::NodeAttrs attrs;
+  op::NumpyReduceAxesBoolParam param;
+
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else if (args[1].type_code() == kDLInt) {
+    param.axis = Tuple<int>(1, args[1].operator int64_t());
+  } else {
+    param.axis = Tuple<int>(args[1].operator ObjectRef());
+  }
+  param.keepdims = args[2].operator bool();
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyReduceAxesBoolParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_percentile_op.cc b/src/api/operator/numpy/np_percentile_op.cc
new file mode 100644
index 000000000000..634ee092c64d
--- /dev/null
+++ b/src/api/operator/numpy/np_percentile_op.cc
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_percentile_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_percentile_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_percentile_op-inl.h"
+
+namespace mxnet {
+
+inline int String2MXNetPercentileType(const std::string& s) {
+  using namespace op;
+  if (s == "linear") {
+    return percentile_enum::kLinear;
+  } else if (s == "lower") {
+    return percentile_enum::kLower;
+  } else if (s == "higher") {
+    return percentile_enum::kHigher;
+  } else if (s == "midpoint") {
+    return percentile_enum::kMidpoint;
+  } else if (s== "nearest") {
+    return percentile_enum::kNearest;
+  } else {
+    LOG(FATAL) << "unknown type " << s;
+  }
+  LOG(FATAL) << "should not reach here ";
+  return 0;
+}
+
+MXNET_REGISTER_API("_npi.percentile")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_percentile");
+  nnvm::NodeAttrs attrs;
+  op::NumpyPercentileParam param;
+
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  if (args[2].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else if (args[2].type_code() == kDLInt) {
+    param.axis = Tuple<int>(1, args[2].operator int64_t());
+  } else {
+    param.axis = Tuple<int>(args[2].operator ObjectRef());
+  }
+  param.interpolation = String2MXNetPercentileType(args[3].operator std::string());
+  param.keepdims = args[4].operator bool();
+  if (args[1].type_code() == kDLInt || args[1].type_code() == kDLFloat) {
+    param.q_scalar = args[1].operator double();
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+    int num_inputs = 1;
+    attrs.parsed = std::move(param);
+    attrs.op = op;
+    SetAttrDict<op::NumpyPercentileParam>(&attrs);
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+    if (out) {
+      *ret = PythonArg(5);
+    } else {
+      *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+    }
+  } else {
+    param.q_scalar = dmlc::nullopt;
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+    int num_inputs = 2;
+    attrs.parsed = std::move(param);
+    attrs.op = op;
+    SetAttrDict<op::NumpyPercentileParam>(&attrs);
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+    if (out) {
+      *ret = PythonArg(5);
+    } else {
+      *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+    }
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/op_utils.cc b/src/api/operator/op_utils.cc
index 220a880336db..bb54662e7a62 100644
--- a/src/api/operator/op_utils.cc
+++ b/src/api/operator/op_utils.cc
@@ -24,6 +24,7 @@
 
 #include "op_utils.h"
 #include <mxnet/base.h>
+#include "../../operator/numpy/np_percentile_op-inl.h"
 
 namespace mxnet {
 
@@ -52,4 +53,24 @@ std::string String2MXNetTypeWithBool(int dtype) {
   return "";
 }
 
+std::string MXNetPercentileType2String(int interpolation) {
+  using namespace op;
+  switch (interpolation) {
+    case percentile_enum::kLinear:
+      return "linear";
+    case percentile_enum::kLower:
+      return "lower";
+    case percentile_enum::kHigher:
+      return "higher";
+    case percentile_enum::kMidpoint:
+      return "midpoint";
+    case percentile_enum::kNearest:
+      return "nearest";
+    default:
+      LOG(FATAL) << "Unknown type enum " << interpolation;
+  }
+  LOG(FATAL) << "should not reach here ";
+  return "";
+}
+
 }  // namespace mxnet
diff --git a/src/api/operator/op_utils.h b/src/api/operator/op_utils.h
index 4c577983c405..f41680df6fd6 100644
--- a/src/api/operator/op_utils.h
+++ b/src/api/operator/op_utils.h
@@ -29,6 +29,7 @@
 namespace mxnet {
 
 std::string String2MXNetTypeWithBool(int dtype);
+std::string MXNetPercentileType2String(int interpolation);
 
 }  // namespace mxnet
 
diff --git a/src/operator/numpy/np_bincount_op-inl.h b/src/operator/numpy/np_bincount_op-inl.h
index 254ea8fdec22..a2e758f36059 100644
--- a/src/operator/numpy/np_bincount_op-inl.h
+++ b/src/operator/numpy/np_bincount_op-inl.h
@@ -28,6 +28,7 @@
 #include <mxnet/operator_util.h>
 #include <utility>
 #include <vector>
+#include <string>
 #include "../mshadow_op.h"
 #include "../mxnet_op.h"
 #include "../operator_common.h"
@@ -50,6 +51,13 @@ struct NumpyBincountParam : public dmlc::Parameter<NumpyBincountParam> {
     .set_default(false)
     .describe("Determine whether Bincount has weights.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream minlength_s, has_weights_s;
+    minlength_s << minlength;
+    has_weights_s << has_weights;
+    (*dict)["minlength"] = minlength_s.str();
+    (*dict)["has_weights"] = has_weights_s.str();
+  }
 };
 
 inline bool NumpyBincountType(const nnvm::NodeAttrs& attrs,
diff --git a/src/operator/numpy/np_broadcast_reduce_op.h b/src/operator/numpy/np_broadcast_reduce_op.h
index 1099d94735d0..52ce81b43bcc 100644
--- a/src/operator/numpy/np_broadcast_reduce_op.h
+++ b/src/operator/numpy/np_broadcast_reduce_op.h
@@ -99,6 +99,13 @@ struct NumpyReduceAxesBoolParam : public dmlc::Parameter<NumpyReduceAxesBoolPara
       .describe("If this is set to `True`, the reduced axes are left "
                 "in the result as dimension with size one.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, keepdims_s;
+    axis_s << axis;
+    keepdims_s << keepdims;
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["keepdims"] = keepdims_s.str();
+  }
 };
 
 inline TShape NumpyReduceAxesShapeImpl(const TShape& ishape,
diff --git a/src/operator/numpy/np_broadcast_reduce_op_boolean.cc b/src/operator/numpy/np_broadcast_reduce_op_boolean.cc
index b137975ca427..0c2f02b0d578 100644
--- a/src/operator/numpy/np_broadcast_reduce_op_boolean.cc
+++ b/src/operator/numpy/np_broadcast_reduce_op_boolean.cc
@@ -39,7 +39,7 @@ inline bool NumpyReduceAxesBoolType(const nnvm::NodeAttrs& attrs,
 
 DMLC_REGISTER_PARAMETER(NumpyReduceAxesBoolParam);
 
-NNVM_REGISTER_OP(_np_any)
+NNVM_REGISTER_OP(_npi_any)
 .add_alias("_np_sometrue")
 .set_attr_parser(ParamParser<NumpyReduceAxesBoolParam>)
 .set_num_inputs(1)
@@ -61,7 +61,7 @@ NNVM_REGISTER_OP(_np_any)
 .add_argument("data", "NDArray-or-Symbol", "Input ndarray")
 .add_arguments(NumpyReduceAxesBoolParam::__FIELDS__());
 
-NNVM_REGISTER_OP(_np_all)
+NNVM_REGISTER_OP(_npi_all)
 .set_attr_parser(ParamParser<NumpyReduceAxesBoolParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
diff --git a/src/operator/numpy/np_broadcast_reduce_op_boolean.cu b/src/operator/numpy/np_broadcast_reduce_op_boolean.cu
index 636c25b505ff..d3247b743bc5 100644
--- a/src/operator/numpy/np_broadcast_reduce_op_boolean.cu
+++ b/src/operator/numpy/np_broadcast_reduce_op_boolean.cu
@@ -28,11 +28,11 @@
 namespace mxnet {
 namespace op {
 
-NNVM_REGISTER_OP(_np_any)
+NNVM_REGISTER_OP(_npi_any)
 .set_attr<FCompute>("FCompute<gpu>", NumpyReduceAxesBoolCompute<gpu,
   mshadow_op::sum, mshadow_op::NonZero, 0>);
 
-NNVM_REGISTER_OP(_np_all)
+NNVM_REGISTER_OP(_npi_all)
 .set_attr<FCompute>("FCompute<gpu>", NumpyReduceAxesBoolCompute<gpu,
   mshadow_op::product, mshadow_op::NonZero, 1>);
 
diff --git a/src/operator/numpy/np_percentile_op-inl.h b/src/operator/numpy/np_percentile_op-inl.h
index 5383adb70ca2..80d275f8872c 100644
--- a/src/operator/numpy/np_percentile_op-inl.h
+++ b/src/operator/numpy/np_percentile_op-inl.h
@@ -25,6 +25,7 @@
 #define MXNET_OPERATOR_NUMPY_NP_PERCENTILE_OP_INL_H_
 
 #include <vector>
+#include <string>
 #include "../tensor/ordering_op-inl.h"
 #include "../tensor/matrix_op-inl.h"
 #include "../../common/utils.h"
@@ -32,6 +33,7 @@
 #include "../operator_common.h"
 #include "../elemwise_op_common.h"
 #include "np_broadcast_reduce_op.h"
+#include "../../api/operator/op_utils.h"
 
 namespace mxnet {
 namespace op {
@@ -65,6 +67,16 @@ struct NumpyPercentileParam : public dmlc::Parameter<NumpyPercentileParam> {
     DMLC_DECLARE_FIELD(q_scalar).set_default(dmlc::optional<double>())
       .describe("inqut q is a scalar");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, keepdims_s, q_scalar_s;
+    axis_s << axis;
+    keepdims_s << keepdims;
+    q_scalar_s << q_scalar;
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["interpolation"] = MXNetPercentileType2String(interpolation);
+    (*dict)["keepdims"] = keepdims_s.str();
+    (*dict)["q_scalar"] = q_scalar_s.str();
+  }
 };
 
 template<int NDim>
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 9cfd6d6c926e..92cd06f3317e 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -7822,10 +7822,8 @@ def hybrid_forward(self, F, a):
         test_diag = TestDiag(k)
         if hybridize:
             test_diag.hybridize()
-
         x = np.random.uniform(-2.0, 2.0, size=shape).astype(dtype) if len(shape) != 0 else np.array(())
         x.attach_grad()
-
         np_out = _np.diag(x.asnumpy(), k)
         with mx.autograd.record():
             mx_out = test_diag(x)

From b88092e97602b55e15c814cff8a9ae19eb589ca1 Mon Sep 17 00:00:00 2001
From: Haozheng Fan <fhztc1997618@gmail.com>
Date: Tue, 10 Mar 2020 13:37:18 +0800
Subject: [PATCH 05/44] Add ffi benchmark (#17780)

Co-authored-by: Haozheng Fan <fanhaozh@amazon.com>
---
 benchmark/python/ffi/benchmark_ffi.py | 123 ++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 benchmark/python/ffi/benchmark_ffi.py

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
new file mode 100644
index 000000000000..88af3cf3d55e
--- /dev/null
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import timeit
+import itertools
+import argparse
+import os
+
+class OpArgMngr(object):
+    """Operator argument manager for storing operator workloads."""
+    args = {}
+
+    @staticmethod
+    def add_workload(funcname, *args, **kwargs):
+        if "_specifier" not in kwargs:
+            _specifier = funcname
+        else:
+            _specifier = kwargs["_specififer"]
+            del kwargs["_specififer"]
+        if _specifier in OpArgMngr.args:
+            raise ValueError("duplicate {}".format(_specifier))
+        OpArgMngr.args[_specifier] = {'args': args, 'kwargs': kwargs, 'funcname': funcname}
+
+
+def generate_workloads():
+    array_pool = {}
+    shapes = []
+    for ndim in range(4):
+        shapes.extend(list(itertools.product(range(4), repeat=ndim)))
+    for shape in shapes:
+        name = 'x'.join(str(i) for i in shape)
+        if name in array_pool:
+            raise ValueError("duplicate array {}".format(name))
+        array_pool[name] = dnp.ones(shape)
+    return array_pool
+
+
+def prepare_workloads():
+    pool = generate_workloads()
+    OpArgMngr.add_workload("zeros", (2, 2))
+    OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
+    OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
+    OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
+
+
+def benchmark_helper(f, *args, **kwargs):
+    number = 10000
+    return timeit.timeit(lambda: f(*args, **kwargs), number=number) / number
+
+
+def get_op(module, funcname):
+    funcname = funcname.split(".")
+    for fname in funcname:
+        module = getattr(module, fname)
+    return module
+
+
+def run_benchmark(packages):
+    results = {}
+    for (k, v) in OpArgMngr.args.items():
+        result = {}
+        for (name, package) in packages.items():
+            print('{}.{} running...'.format(name, k))
+            op = get_op(package["module"], v["funcname"])
+            args = [package["data"](arg) for arg in v["args"]]
+            kwargs = {k: package["data"](v) for (k, v) in v["kwargs"].items()}
+            benchmark = benchmark_helper(op, *args, **kwargs)
+            result[name] = benchmark
+        results[k] = result
+    return results
+
+
+def show_results(results):
+    print("{:>24}{:>24}{:>24}".format("name", "package", "time(us)"))
+    for (specifier, d) in results.items():
+        for (k, v) in d.items():
+            print("{:>24}{:>24}{:>24}".format(specifier, k, v * 10 ** 6))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('ffi_type')
+    parsed = parser.parse_args()
+    if parsed.ffi_type == "cython":
+        os.environ['MXNET_ENABLE_CYTHON'] = '1'
+        os.environ['MXNET_ENFORCE_CYTHON'] = '1'
+    elif parsed.ffi_type == "ctypes":
+        os.environ['MXNET_ENABLE_CYTHON'] = '0'
+    else:
+        raise ValueError("unknown ffi_type {}",format(parsed.ffi_type))
+    os.environ["MXNET_ENGINE_TYPE"] = "NaiveEngine"
+    import mxnet as mx
+    import numpy as onp
+    from mxnet import np as dnp
+
+    mx.npx.set_np()
+    packages = {
+        "onp": {
+            "module": onp,
+            "data": lambda arr: arr.asnumpy() if isinstance(arr, dnp.ndarray) else arr
+        },
+        "dnp": {
+            "module": dnp,
+            "data": lambda arr: arr
+        }
+    }
+    prepare_workloads()
+    results = run_benchmark(packages)
+    show_results(results)

From 2a3dbdab80f13b3ec56708eb18a2e55c29dab5d2 Mon Sep 17 00:00:00 2001
From: alicia <32725332+Alicia1529@users.noreply.github.com>
Date: Wed, 18 Mar 2020 05:05:34 +0800
Subject: [PATCH 06/44] ffi wrappers for polyval, ediff1d, nan_to_num (#17832)

---
 benchmark/python/ffi/benchmark_ffi.py      |  3 +
 python/mxnet/ndarray/numpy/_op.py          | 29 ++-------
 src/api/operator/numpy/np_ediff1d_op.cc    | 75 ++++++++++++++++++++++
 src/api/operator/numpy/np_nan_to_num_op.cc | 72 +++++++++++++++++++++
 src/api/operator/numpy/np_polynomial_op.cc | 44 +++++++++++++
 src/operator/numpy/np_ediff1d_op-inl.h     | 13 ++++
 src/operator/tensor/elemwise_unary_op.h    | 12 ++++
 7 files changed, 225 insertions(+), 23 deletions(-)
 create mode 100644 src/api/operator/numpy/np_ediff1d_op.cc
 create mode 100644 src/api/operator/numpy/np_nan_to_num_op.cc
 create mode 100644 src/api/operator/numpy/np_polynomial_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 88af3cf3d55e..e717746cff6a 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -51,6 +51,9 @@ def generate_workloads():
 def prepare_workloads():
     pool = generate_workloads()
     OpArgMngr.add_workload("zeros", (2, 2))
+    OpArgMngr.add_workload("polyval", dnp.arange(10), pool['2x2'])
+    OpArgMngr.add_workload("ediff1d", pool['2x2'], pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("nan_to_num", pool['2x2'])
     OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index b29586fca190..16bd4376d6ce 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -6983,24 +6983,7 @@ def ediff1d(ary, to_end=None, to_begin=None):
     >>> np.ediff1d(x, to_begin=y)
     array([ 1.,  2.,  4.,  1.,  6., 24.,  1.,  2.,  3., -7.])
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(to_begin, np_ndarray), isinstance(to_end, np_ndarray))
-    # case 1: when both `to_begin` and `to_end` are arrays
-    if input_type == (True, True):
-        return _npi.ediff1d(ary, to_begin, to_end, to_begin_arr_given=True, to_end_arr_given=True,
-                            to_begin_scalar=None, to_end_scalar=None)
-    # case 2: only `to_end` is array but `to_begin` is scalar/None
-    elif input_type == (False, True):
-        return _npi.ediff1d(ary, to_end, to_begin_arr_given=False, to_end_arr_given=True,
-                            to_begin_scalar=to_begin, to_end_scalar=None)
-    # case 3: only `to_begin` is array but `to_end` is scalar/None
-    elif input_type == (True, False):
-        return _npi.ediff1d(ary, to_begin, to_begin_arr_given=True, to_end_arr_given=False,
-                            to_begin_scalar=None, to_end_scalar=to_end)
-    # case 4: both `to_begin` and `to_end` are scalar/None
-    else:
-        return _npi.ediff1d(ary, to_begin_arr_given=False, to_end_arr_given=False,
-                            to_begin_scalar=to_begin, to_end_scalar=to_end)
+    return _api_internal.ediff1d(ary, to_end, to_begin)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7148,8 +7131,8 @@ def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None, **kwargs):
         if x.dtype in ['int8', 'uint8', 'int32', 'int64']:
             return x
         if not copy:
-            return _npi.nan_to_num(x, copy=copy, nan=nan, posinf=posinf, neginf=neginf, out=x)
-        return _npi.nan_to_num(x, copy=copy, nan=nan, posinf=posinf, neginf=neginf, out=None)
+            return _api_internal.nan_to_num(x, copy, nan, posinf, neginf, x)
+        return _api_internal.nan_to_num(x, copy, nan, posinf, neginf, None)
     else:
         raise TypeError('type {} not supported'.format(str(type(x))))
 
@@ -7538,10 +7521,10 @@ def polyval(p, x):
     array([76., 49.])
     """
     from ...numpy import ndarray
-    if isinstance(p, ndarray) and isinstance(x, ndarray):
-        return _npi.polyval(p, x)
-    elif not isinstance(p, ndarray) and not isinstance(x, ndarray):
+    if isinstance(p, numeric_types) and isinstance(x, numeric_types):
         return _np.polyval(p, x)
+    elif isinstance(p, ndarray) and isinstance(x, ndarray):
+        return _api_internal.polyval(p, x)
     else:
         raise TypeError('type not supported')
 
diff --git a/src/api/operator/numpy/np_ediff1d_op.cc b/src/api/operator/numpy/np_ediff1d_op.cc
new file mode 100644
index 000000000000..df97fd8b68b9
--- /dev/null
+++ b/src/api/operator/numpy/np_ediff1d_op.cc
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_ediff1d_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_ediff1d_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_ediff1d_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.ediff1d")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_ediff1d");
+  nnvm::NodeAttrs attrs;
+  op::EDiff1DParam param;
+  int num_inputs = 1;
+  NDArray* inputs[3];
+  inputs[0] = args[0].operator mxnet::NDArray*();
+  // the order of `to_end` and `to_begin` array in the backend is different from the front-end
+  if (args[2].type_code() == kDLFloat || args[2].type_code() == kDLInt) {
+    param.to_begin_scalar = args[2].operator double();
+    param.to_begin_arr_given = false;
+  } else if (args[2].type_code() == kNull) {
+    param.to_begin_scalar = dmlc::nullopt;
+    param.to_begin_arr_given = false;
+  } else {
+    param.to_begin_scalar = dmlc::nullopt;
+    param.to_begin_arr_given = true;
+    inputs[num_inputs] = args[2].operator mxnet::NDArray*();
+    num_inputs++;
+  }
+
+  if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+    param.to_end_scalar = args[1].operator double();
+    param.to_end_arr_given = false;
+  } else if (args[1].type_code() == kNull) {
+    param.to_end_scalar = dmlc::nullopt;
+    param.to_end_arr_given = false;
+  } else {
+    param.to_end_scalar = dmlc::nullopt;
+    param.to_end_arr_given = true;
+    inputs[num_inputs] = args[1].operator mxnet::NDArray*();
+    num_inputs++;
+  }
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::EDiff1DParam>(&attrs);
+
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_nan_to_num_op.cc b/src/api/operator/numpy/np_nan_to_num_op.cc
new file mode 100644
index 000000000000..fadc4fe55dc7
--- /dev/null
+++ b/src/api/operator/numpy/np_nan_to_num_op.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_nan_to_num_op.cc
+ * \brief Implementation of the API of nan_to_num function in
+ *        src/operator/tensor/np_elemwise_unary_op_basic.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/tensor/elemwise_unary_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.nan_to_num")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_nan_to_num");
+  nnvm::NodeAttrs attrs;
+
+  op::NumpyNanToNumParam param;
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+
+  param.copy = args[1].operator bool();
+  param.nan = args[2].operator double();
+
+  if (args[3].type_code() == kNull) {
+    param.posinf = dmlc::nullopt;
+  } else {
+    param.posinf = args[3].operator double();
+  }
+
+  if (args[4].type_code() == kNull) {
+    param.neginf = dmlc::nullopt;
+  } else {
+    param.neginf = args[4].operator double();
+  }
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyNanToNumParam>(&attrs);
+
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  // set the number of outputs provided by the `out` arugment
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_polynomial_op.cc b/src/api/operator/numpy/np_polynomial_op.cc
new file mode 100644
index 000000000000..87081d2952ca
--- /dev/null
+++ b/src/api/operator/numpy/np_polynomial_op.cc
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_polynomial_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_polynomial_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_polynomial_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.polyval")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_polyval");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  int num_inputs = 2;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_ediff1d_op-inl.h b/src/operator/numpy/np_ediff1d_op-inl.h
index f1d5c2998000..f43040ceb25a 100644
--- a/src/operator/numpy/np_ediff1d_op-inl.h
+++ b/src/operator/numpy/np_ediff1d_op-inl.h
@@ -28,6 +28,7 @@
 #include <mxnet/base.h>
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../mxnet_op.h"
 #include "../operator_common.h"
 #include "../elemwise_op_common.h"
@@ -53,6 +54,18 @@ struct EDiff1DParam : public dmlc::Parameter<EDiff1DParam> {
       .set_default(dmlc::optional<double>())
       .describe("If the `to_end`is a scalar, the value of this parameter.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream to_end_arr_given_s, to_begin_arr_given_s,
+                       to_end_scalar_s, to_begin_scalar_s;
+    to_end_arr_given_s << to_end_arr_given;
+    to_begin_arr_given_s << to_begin_arr_given;
+    to_end_scalar_s << to_end_scalar;
+    to_begin_scalar_s << to_begin_scalar;
+    (*dict)["to_end_arr_given"] = to_end_arr_given_s.str();
+    (*dict)["to_begin_arr_given"] = to_begin_arr_given_s.str();
+    (*dict)["to_end_scalar"] = to_end_scalar_s.str();
+    (*dict)["to_begin_scalar"] = to_begin_scalar_s.str();
+  }
 };
 
 template<typename DType>
diff --git a/src/operator/tensor/elemwise_unary_op.h b/src/operator/tensor/elemwise_unary_op.h
index dcbd53aac69b..49848caccca8 100644
--- a/src/operator/tensor/elemwise_unary_op.h
+++ b/src/operator/tensor/elemwise_unary_op.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include <utility>
 #include <algorithm>
 #include <climits>
@@ -704,6 +705,17 @@ struct NumpyNanToNumParam : public dmlc::Parameter<NumpyNanToNumParam> {
               "If no value is passed then negative infinity values"
               "will be replaced with a very small (or negative) number.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream copy_s, nan_s, posinf_s, neginf_s;
+    copy_s << copy;
+    nan_s << nan;
+    posinf_s << posinf;
+    neginf_s << neginf;
+    (*dict)["copy"] = copy_s.str();
+    (*dict)["nan"] = nan_s.str();
+    (*dict)["posinf"] = posinf_s.str();
+    (*dict)["neginf"] = neginf_s.str();
+  }
 };
 
 template<int req>

From 8750350f46783e501a40dfad552a04f92287b583 Mon Sep 17 00:00:00 2001
From: Haozheng Fan <fhztc1997618@gmail.com>
Date: Sat, 29 Feb 2020 02:20:05 +0800
Subject: [PATCH 07/44] Fix compiler warnings in new FFI (#17718)

Introduced in https://github.com/apache/incubator-mxnet/pull/17510
---
 include/mxnet/ir/expr.h            |  4 +-
 include/mxnet/node/container.h     |  2 +-
 include/mxnet/runtime/container.h  |  4 +-
 include/mxnet/runtime/ffi_helper.h | 10 ++---
 include/mxnet/runtime/object.h     | 66 +++++++++++++++---------------
 5 files changed, 42 insertions(+), 44 deletions(-)

diff --git a/include/mxnet/ir/expr.h b/include/mxnet/ir/expr.h
index b9483c74320a..a9f4ff2bbf70 100644
--- a/include/mxnet/ir/expr.h
+++ b/include/mxnet/ir/expr.h
@@ -141,7 +141,7 @@ class IntImmNode : public PrimExprNode {
   int64_t value;
 
   static constexpr const char* _type_key = "IntImm";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(IntImmNode, PrimExprNode);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(IntImmNode, PrimExprNode)
 };
 
 /*!
@@ -186,7 +186,7 @@ class FloatImmNode : public PrimExprNode {
   double value;
 
   static constexpr const char* _type_key = "FloatImm";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(FloatImmNode, PrimExprNode);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(FloatImmNode, PrimExprNode)
 };
 
 /*!
diff --git a/include/mxnet/node/container.h b/include/mxnet/node/container.h
index 27b9853a74b7..e164f64a9184 100644
--- a/include/mxnet/node/container.h
+++ b/include/mxnet/node/container.h
@@ -42,7 +42,7 @@ class ArrayNode : public Object {
   std::vector<ObjectRef> data;
 
   static constexpr const char* _type_key = "Array";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(ArrayNode, Object);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(ArrayNode, Object)
 };
 
 /*!
diff --git a/include/mxnet/runtime/container.h b/include/mxnet/runtime/container.h
index 3dd7e0fc9c79..cd719aaa51a6 100644
--- a/include/mxnet/runtime/container.h
+++ b/include/mxnet/runtime/container.h
@@ -173,7 +173,7 @@ class ADTObj : public Object, public InplaceArrayBase<ADTObj, ObjectRef> {
 
   static constexpr const uint32_t _type_index = TypeIndex::kMXNetADT;
   static constexpr const char* _type_key = "MXNet.ADT";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(ADTObj, Object);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(ADTObj, Object)
 
  private:
   /*!
@@ -273,7 +273,7 @@ class ADT : public ObjectRef {
     return ADT(0, std::forward<Args>(args)...);
   }
 
-  MXNET_DEFINE_OBJECT_REF_METHODS(ADT, ObjectRef, ADTObj);
+  MXNET_DEFINE_OBJECT_REF_METHODS(ADT, ObjectRef, ADTObj)
 };
 
 }  // namespace runtime
diff --git a/include/mxnet/runtime/ffi_helper.h b/include/mxnet/runtime/ffi_helper.h
index b539524dfd05..49134ca122a7 100644
--- a/include/mxnet/runtime/ffi_helper.h
+++ b/include/mxnet/runtime/ffi_helper.h
@@ -38,7 +38,7 @@ class EllipsisObj : public Object {
  public:
   static constexpr const uint32_t _type_index = TypeIndex::kEllipsis;
   static constexpr const char* _type_key = "MXNet.Ellipsis";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(EllipsisObj, Object);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(EllipsisObj, Object)
 };
 
 inline ObjectRef CreateEllipsis() {
@@ -54,7 +54,7 @@ class SliceObj : public Object {
 
   static constexpr const uint32_t _type_index = TypeIndex::kSlice;
   static constexpr const char* _type_key = "MXNet.Slice";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(SliceObj, Object);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(SliceObj, Object)
 };
 
 class Slice : public ObjectRef {
@@ -74,7 +74,7 @@ class Slice : public ObjectRef {
   // constant to represent None.
   static constexpr int64_t kNoneValue = std::numeric_limits<int64_t>::min();
 
-  MXNET_DEFINE_OBJECT_REF_METHODS(Slice, ObjectRef, SliceObj);
+  MXNET_DEFINE_OBJECT_REF_METHODS(Slice, ObjectRef, SliceObj)
 };
 
 int64_t inline SliceNoneValue() {
@@ -86,7 +86,7 @@ class IntegerObj: public Object {
   int64_t value;
   static constexpr const uint32_t _type_index = TypeIndex::kInteger;
   static constexpr const char* _type_key = "MXNet.Integer";
-  MXNET_DECLARE_FINAL_OBJECT_INFO(IntegerObj, Object);
+  MXNET_DECLARE_FINAL_OBJECT_INFO(IntegerObj, Object)
 };
 
 class Integer: public ObjectRef {
@@ -96,7 +96,7 @@ class Integer: public ObjectRef {
     data->value = value;
     data_ = std::move(data);
   }
-  MXNET_DEFINE_OBJECT_REF_METHODS(Integer, ObjectRef, IntegerObj);
+  MXNET_DEFINE_OBJECT_REF_METHODS(Integer, ObjectRef, IntegerObj)
 };
 
 //  Helper functions for fast FFI implementations
diff --git a/include/mxnet/runtime/object.h b/include/mxnet/runtime/object.h
index e2fb067f1067..a031a56d88ed 100644
--- a/include/mxnet/runtime/object.h
+++ b/include/mxnet/runtime/object.h
@@ -644,22 +644,20 @@ struct ObjectEqual {
  * \param TypeName The name of the current type.
  * \param ParentType The name of the ParentType
  */
-#define MXNET_DECLARE_BASE_OBJECT_INFO(TypeName, ParentType)              \
-  static const uint32_t RuntimeTypeIndex()  {                           \
-    if (TypeName::_type_index != ::mxnet::runtime::TypeIndex::kDynamic) { \
-      return TypeName::_type_index;                                     \
-    }                                                                   \
-    return _GetOrAllocRuntimeTypeIndex();                               \
-  }                                                                     \
-  static const uint32_t _GetOrAllocRuntimeTypeIndex()  {                \
-    static uint32_t tidx = GetOrAllocRuntimeTypeIndex(                  \
-        TypeName::_type_key,                                            \
-        TypeName::_type_index,                                          \
-        ParentType::_GetOrAllocRuntimeTypeIndex(),                      \
-        TypeName::_type_child_slots,                                    \
-        TypeName::_type_child_slots_can_overflow);                      \
-    return tidx;                                                        \
-  }                                                                     \
+#define MXNET_DECLARE_BASE_OBJECT_INFO(TypeName, ParentType)                \
+  static uint32_t RuntimeTypeIndex()  {                                     \
+    return TypeName::_type_index != ::mxnet::runtime::TypeIndex::kDynamic ? \
+           TypeName::_type_index : _GetOrAllocRuntimeTypeIndex();           \
+  }                                                                         \
+  static uint32_t _GetOrAllocRuntimeTypeIndex()  {                          \
+    static uint32_t tidx = GetOrAllocRuntimeTypeIndex(                      \
+        TypeName::_type_key,                                                \
+        TypeName::_type_index,                                              \
+        ParentType::_GetOrAllocRuntimeTypeIndex(),                          \
+        TypeName::_type_child_slots,                                        \
+        TypeName::_type_child_slots_can_overflow);                          \
+    return tidx;                                                            \
+  }
 
 /*!
  * \brief helper macro to declare type information in a final class.
@@ -667,8 +665,8 @@ struct ObjectEqual {
   * \param ParentType The name of the ParentType
   */
 #define MXNET_DECLARE_FINAL_OBJECT_INFO(TypeName, ParentType)             \
-  static const constexpr bool _type_final = true;                       \
-  static const constexpr int _type_child_slots = 0;                     \
+  static const constexpr bool _type_final = true;                         \
+  static const constexpr int _type_child_slots = 0;                       \
   MXNET_DECLARE_BASE_OBJECT_INFO(TypeName, ParentType)                    \
 
 
@@ -684,25 +682,25 @@ struct ObjectEqual {
 
 
 #define MXNET_DEFINE_OBJECT_REF_METHODS(TypeName, ParentType, ObjectName) \
-  TypeName() {}                                                         \
-  explicit TypeName(                                                    \
-      ::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n)              \
-      : ParentType(n) {}                                                \
-  const ObjectName* operator->() const {                                \
-    return static_cast<const ObjectName*>(data_.get());                 \
-  }                                                                     \
-  operator bool() const { return data_ != nullptr; }                    \
+  TypeName() {}                                                           \
+  explicit TypeName(                                                      \
+      ::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n)            \
+      : ParentType(n) {}                                                  \
+  const ObjectName* operator->() const {                                  \
+    return static_cast<const ObjectName*>(data_.get());                   \
+  }                                                                       \
+  operator bool() const { return data_ != nullptr; }                      \
   using ContainerType = ObjectName;
 
 #define MXNET_DEFINE_OBJECT_REF_METHODS_MUT(TypeName, ParentType, ObjectName) \
-  TypeName() {}                                                             \
-  explicit TypeName(                                                        \
-      ::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n)                  \
-      : ParentType(n) {}                                                    \
-  ObjectName* operator->() {                                    \
-    return static_cast<ObjectName*>(data_.get());                     \
-  }                                                                         \
-  operator bool() const { return data_ != nullptr; }                        \
+  TypeName() {}                                                               \
+  explicit TypeName(                                                          \
+      ::mxnet::runtime::ObjectPtr<::mxnet::runtime::Object> n)                \
+      : ParentType(n) {}                                                      \
+  ObjectName* operator->() {                                                  \
+    return static_cast<ObjectName*>(data_.get());                             \
+  }                                                                           \
+  operator bool() const { return data_ != nullptr; }                          \
   using ContainerType = ObjectName;
 
 // Implementations details below

From 8c900402cc05c9f0d344dc75f6db75c182d51193 Mon Sep 17 00:00:00 2001
From: alicia <32725332+Alicia1529@users.noreply.github.com>
Date: Tue, 10 Mar 2020 06:28:04 +0800
Subject: [PATCH 08/44] ffi invocation: expand_dims, tril, diff, broadcast_to
 (#17738)

---
 python/mxnet/ndarray/numpy/_op.py             | 10 ++--
 .../numpy/np_broadcast_reduce_op_value.cc     | 53 +++++++++++++++++++
 src/api/operator/numpy/np_diff_op.cc          | 50 +++++++++++++++++
 src/api/operator/numpy/np_matrix_op.cc        | 49 +++++++++++++++++
 src/api/operator/numpy/np_tril_op.cc          | 49 +++++++++++++++++
 src/operator/numpy/np_diff-inl.h              | 10 +++-
 src/operator/numpy/np_tril_op-inl.h           |  6 +++
 src/operator/tensor/broadcast_reduce_op.h     |  5 ++
 src/operator/tensor/matrix_op-inl.h           |  7 +++
 9 files changed, 232 insertions(+), 7 deletions(-)
 create mode 100644 src/api/operator/numpy/np_broadcast_reduce_op_value.cc
 create mode 100644 src/api/operator/numpy/np_diff_op.cc
 create mode 100644 src/api/operator/numpy/np_matrix_op.cc
 create mode 100644 src/api/operator/numpy/np_tril_op.cc

diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 16bd4376d6ce..ae53645bb4e0 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -304,7 +304,7 @@ def broadcast_to(array, shape):
     """
     if _np.isscalar(array):
         return full(shape, array)
-    return _npi.broadcast_to(array, shape)
+    return _api_internal.broadcast_to(array, shape)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1938,7 +1938,7 @@ def expand_dims(a, axis):
         Output array. The number of dimensions is one greater than that of
         the input array.
     """
-    return _npi.expand_dims(a, axis)
+    return _api_internal.expand_dims(a, axis)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -2012,7 +2012,7 @@ def tril(m, k=0):
            [ 7.,  8.,  0.],
            [10., 11., 12.]])
     """
-    return _npi.tril(m, k)
+    return _api_internal.tril(m, k)
 
 
 def _unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):
@@ -6933,7 +6933,7 @@ def diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefin
     >>> x = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])
     >>> np.diff(x)
     array([[2, 3, 4],
-        [5, 1, 2]])
+           [5, 1, 2]])
     >>> np.diff(x, axis=0)
     array([[-1,  2,  0, -2]])
 
@@ -6943,7 +6943,7 @@ def diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefin
     """
     if (prepend or append):
         raise NotImplementedError('prepend and append options are not supported yet')
-    return _npi.diff(a, n=n, axis=axis)
+    return _api_internal.diff(a, n, axis)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
new file mode 100644
index 000000000000..2322860aa609
--- /dev/null
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file broadcast_reduce_op_value.cc
+ * \brief Implementation of the API of functions in
+ * src/operator/tensor/np_broadcast_reduce_op_value.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/tensor/broadcast_reduce_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.broadcast_to")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_broadcast_to");
+  nnvm::NodeAttrs attrs;
+  op::BroadcastToParam param;
+  if (args[1].type_code() == kDLInt) {
+    param.shape = TShape(1, args[1].operator int64_t());
+  } else {
+    param.shape = TShape(args[1].operator ObjectRef());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::BroadcastToParam>(&attrs);
+
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_diff_op.cc b/src/api/operator/numpy/np_diff_op.cc
new file mode 100644
index 000000000000..dec73b8496a7
--- /dev/null
+++ b/src/api/operator/numpy/np_diff_op.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_diff_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_diff.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_diff-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.diff")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_diff");
+  nnvm::NodeAttrs attrs;
+  op::DiffParam param;
+  param.n = args[1].operator int();
+  param.axis = args[2].operator int();
+
+  // we directly copy DiffParam, which is trivially-copyable
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::DiffParam>(&attrs);
+
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
new file mode 100644
index 000000000000..080cca867ecb
--- /dev/null
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_matrix_op.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/matrix_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/tensor/matrix_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.expand_dims")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_expand_dims");
+  nnvm::NodeAttrs attrs;
+  op::ExpandDimParam param;
+  param.axis = args[1].operator int();
+
+  // we directly copy ExpandDimParam, which is trivially-copyable
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::ExpandDimParam>(&attrs);
+
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_tril_op.cc b/src/api/operator/numpy/np_tril_op.cc
new file mode 100644
index 000000000000..105ff58bf559
--- /dev/null
+++ b/src/api/operator/numpy/np_tril_op.cc
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_tril_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_diff.cc
+ */
+#include <mxnet/api_registry.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_tril_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.tril")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_tril");
+  nnvm::NodeAttrs attrs;
+  op::TrilParam param;
+  param.k = args[1].operator int();
+
+  // we directly copy TrilParam, which is trivially-copyable
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::TrilParam>(&attrs);
+
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_diff-inl.h b/src/operator/numpy/np_diff-inl.h
index 69f175e802dd..8a8bc558962a 100644
--- a/src/operator/numpy/np_diff-inl.h
+++ b/src/operator/numpy/np_diff-inl.h
@@ -28,6 +28,7 @@
 #include <mxnet/base.h>
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../mxnet_op.h"
 #include "../operator_common.h"
 #include "../tensor/broadcast_reduce_op.h"
@@ -37,8 +38,6 @@ namespace op {
 
 struct DiffParam : public dmlc::Parameter<DiffParam> {
   int n, axis;
-  dmlc::optional<mxnet::TShape> prepend;
-  dmlc::optional<mxnet::TShape> append;
   DMLC_DECLARE_PARAMETER(DiffParam) {
     DMLC_DECLARE_FIELD(n).set_default(1).describe(
         "The number of times values are differenced."
@@ -47,6 +46,13 @@ struct DiffParam : public dmlc::Parameter<DiffParam> {
         "Axis along which the cumulative sum is computed."
         " The default (None) is to compute the diff over the flattened array.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream n_s, axis_s;
+    n_s << n;
+    axis_s << axis;
+    (*dict)["n"] = n_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 inline void YanghuiTri(std::vector<int>* buffer, int n) {
diff --git a/src/operator/numpy/np_tril_op-inl.h b/src/operator/numpy/np_tril_op-inl.h
index 1ad74e887b6c..50943a09caa2 100644
--- a/src/operator/numpy/np_tril_op-inl.h
+++ b/src/operator/numpy/np_tril_op-inl.h
@@ -28,6 +28,7 @@
 
 #include <dmlc/parameter.h>
 #include <vector>
+#include <string>
 #include <algorithm>
 #include "../mxnet_op.h"
 #include "../operator_common.h"
@@ -46,6 +47,11 @@ struct TrilParam : public dmlc::Parameter<TrilParam> {
                 "and k<0 for diagonals below the main diagonal. "
                 "If input has shape (S0 S1) k must be between -S0 and S1");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream k_s;
+    k_s << k;
+    (*dict)["k"] = k_s.str();
+  }
 };
 
 inline bool TrilOpShape(const nnvm::NodeAttrs& attrs,
diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h
index 5eb0c41aa36c..b06442932a78 100644
--- a/src/operator/tensor/broadcast_reduce_op.h
+++ b/src/operator/tensor/broadcast_reduce_op.h
@@ -156,6 +156,11 @@ struct BroadcastToParam : public dmlc::Parameter<BroadcastToParam> {
                 " E.g `A = broadcast_to(B, shape=(10, 0, 0))` "
                 "has the same meaning as `A = broadcast_axis(B, axis=0, size=10)`.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream shape_s;
+    shape_s << shape;
+    (*dict)["shape"] = shape_s.str();
+  }
 };
 
 struct BroadcastLikeParam : public dmlc::Parameter<BroadcastLikeParam> {
diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h
index fa7b8a10b212..670104bcfdb0 100644
--- a/src/operator/tensor/matrix_op-inl.h
+++ b/src/operator/tensor/matrix_op-inl.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include <algorithm>
 #include <utility>
 #include <type_traits>
@@ -497,6 +498,12 @@ struct ExpandDimParam : public dmlc::Parameter<ExpandDimParam> {
   bool operator==(const ExpandDimParam &other) const {
     return this->axis == other.axis;
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s;
+    axis_s << axis;
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 

From e4062939b3ad79244a10b866c7f51c39e19937ce Mon Sep 17 00:00:00 2001
From: Haozheng Fan <fhztc1997618@gmail.com>
Date: Wed, 18 Mar 2020 05:06:01 +0800
Subject: [PATCH 09/44] [Numpy] FFI for split and svd (#17816)

* Support ADT as FFI return value

* Special operator= for NDArrayHandle

* SVD

* Support cython

* Clear

* Add split

* Refine

* Fix ci

* Fix typo

* Clear

* Resolve sanity issues

Co-authored-by: Haozheng Fan <fanhaozh@amazon.com>
---
 benchmark/python/ffi/benchmark_ffi.py     |   2 +
 include/mxnet/runtime/c_runtime_api.h     |  18 ++++
 include/mxnet/runtime/container.h         |   2 +-
 include/mxnet/runtime/ndarray_handle.h    |  52 +++++++++++
 include/mxnet/runtime/packed_func.h       |  11 ++-
 python/mxnet/__init__.py                  |   1 +
 python/mxnet/_ffi/_ctypes/function.py     |  23 +++--
 python/mxnet/_ffi/_ctypes/object.py       |  47 +++++++++-
 python/mxnet/_ffi/_cython/base.pxi        |   2 +
 python/mxnet/_ffi/_cython/core.pyx        |   1 +
 python/mxnet/_ffi/_cython/function.pxi    |  26 ++++++
 python/mxnet/_ffi/_cython/object.pxi      |  98 +++++++++++++++++++++
 python/mxnet/_ffi/object.py               | 101 +++++++++++++++++++++-
 python/mxnet/container.py                 |  54 ++++++++++++
 python/mxnet/ndarray/numpy/_op.py         |  20 +----
 python/mxnet/ndarray/numpy/linalg.py      |   3 +-
 src/api/operator/numpy/linalg/np_gesvd.cc |  47 ++++++++++
 src/api/operator/numpy/np_matrix_op.cc    |  39 +++++++++
 src/api/operator/numpy/np_tensordot_op.cc |   8 +-
 src/api/operator/op_utils.cc              |   2 +-
 src/api/operator/op_utils.h               |   2 +-
 src/operator/numpy/np_cumsum-inl.h        |   2 +-
 src/operator/numpy/np_cumsum.cc           |   1 +
 src/operator/tensor/init_op.h             |   6 +-
 src/operator/tensor/matrix_op-inl.h       |  11 +++
 src/runtime/container.cc                  |  72 +++++++++++++++
 src/runtime/ndarray_handle.cc             |  42 +++++++++
 src/runtime/object.cc                     |  14 +++
 src/runtime/object_internal.h             |   9 ++
 29 files changed, 678 insertions(+), 38 deletions(-)
 create mode 100644 include/mxnet/runtime/ndarray_handle.h
 create mode 100644 python/mxnet/_ffi/_cython/object.pxi
 create mode 100644 python/mxnet/container.py
 create mode 100644 src/api/operator/numpy/linalg/np_gesvd.cc
 create mode 100644 src/runtime/container.cc
 create mode 100644 src/runtime/ndarray_handle.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index e717746cff6a..c420ee9cdad3 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -57,6 +57,8 @@ def prepare_workloads():
     OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("linalg.svd", pool['3x3'])
+    OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
 
 
diff --git a/include/mxnet/runtime/c_runtime_api.h b/include/mxnet/runtime/c_runtime_api.h
index bbc8862d5439..69de9ca27d12 100644
--- a/include/mxnet/runtime/c_runtime_api.h
+++ b/include/mxnet/runtime/c_runtime_api.h
@@ -156,6 +156,24 @@ MXNET_DLL int MXNetFuncListGlobalNames(int* out_size,
  */
 MXNET_DLL int MXNetObjectFree(MXNetObjectHandle obj);
 
+
+/*!
+ * \brief Get the type_index from an object.
+ *
+ * \param obj The object handle.
+ * \param out_tindex the output type index.
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXNetObjectGetTypeIndex(MXNetObjectHandle obj, unsigned* out_tindex);
+
+/*!
+ * \brief Convert type key to type index.
+ * \param type_key The key of the type.
+ * \param out_tindex the corresponding type index.
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXNetObjectTypeKey2Index(const char* type_key, unsigned* out_tindex);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/include/mxnet/runtime/container.h b/include/mxnet/runtime/container.h
index cd719aaa51a6..fc1d4a173669 100644
--- a/include/mxnet/runtime/container.h
+++ b/include/mxnet/runtime/container.h
@@ -171,8 +171,8 @@ class ADTObj : public Object, public InplaceArrayBase<ADTObj, ObjectRef> {
   uint32_t size{0};
   // The fields of the structure follows directly in memory.
 
-  static constexpr const uint32_t _type_index = TypeIndex::kMXNetADT;
   static constexpr const char* _type_key = "MXNet.ADT";
+  static constexpr const uint32_t _type_index = TypeIndex::kMXNetADT;
   MXNET_DECLARE_FINAL_OBJECT_INFO(ADTObj, Object)
 
  private:
diff --git a/include/mxnet/runtime/ndarray_handle.h b/include/mxnet/runtime/ndarray_handle.h
new file mode 100644
index 000000000000..e6da83c0c213
--- /dev/null
+++ b/include/mxnet/runtime/ndarray_handle.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ndarray_handle.h
+ * \brief NDArray handle types
+ */
+#ifndef MXNET_RUNTIME_NDARRAY_HANDLE_H_
+#define MXNET_RUNTIME_NDARRAY_HANDLE_H_
+#include <mxnet/ndarray.h>
+#include <mxnet/runtime/object.h>
+
+namespace mxnet {
+
+class NDArrayHandleObj : public Object {
+ public:
+  /*! \brief the Internal value. */
+  NDArray* value;
+
+  static constexpr const char* _type_key = "MXNet.NDArrayHandle";
+  MXNET_DECLARE_FINAL_OBJECT_INFO(NDArrayHandleObj, Object)
+};
+
+class NDArrayHandle : public ObjectRef {
+ public:
+  explicit NDArrayHandle(NDArray* value) {
+    runtime::ObjectPtr<NDArrayHandleObj> node = make_object<NDArrayHandleObj>();
+    node->value = value;
+    data_ = std::move(node);
+  }
+  MXNET_DEFINE_OBJECT_REF_METHODS(NDArrayHandle, ObjectRef, NDArrayHandleObj)
+};
+
+};  // namespace mxnet
+
+#endif  // MXNET_RUNTIME_NDARRAY_HANDLE_H_
diff --git a/include/mxnet/runtime/packed_func.h b/include/mxnet/runtime/packed_func.h
index ac7b462ce471..066ec8538338 100644
--- a/include/mxnet/runtime/packed_func.h
+++ b/include/mxnet/runtime/packed_func.h
@@ -30,6 +30,7 @@
 #include <mxnet/runtime/object.h>
 #include <mxnet/runtime/ndarray.h>
 #include <mxnet/runtime/container.h>
+#include <mxnet/runtime/ndarray_handle.h>
 #include <mxnet/runtime/ffi_helper.h>
 #include <mxnet/runtime/data_type.h>
 #include <mxnet/runtime/py_arg.h>
@@ -651,6 +652,9 @@ class MXNetRetValue : public MXNetPODValue_ {
     return *this;
   }
   MXNetRetValue& operator=(ObjectRef other) {
+    if (other.as<NDArrayHandleObj>()) {
+      return operator=(Downcast<NDArrayHandle, ObjectRef>(other));
+    }
     return operator=(std::move(other.data_));
   }
   template<typename T>
@@ -670,11 +674,16 @@ class MXNetRetValue : public MXNetPODValue_ {
     this->Assign(other);
     return *this;
   }
-  MXNetRetValue& operator=(::mxnet::NDArray* value) {
+  MXNetRetValue& operator=(NDArray* value) {
     this->SwitchToPOD(kNDArrayHandle);
     value_.v_handle = reinterpret_cast<void*>(value);
     return *this;
   }
+  MXNetRetValue& operator=(NDArrayHandle value) {
+    this->SwitchToPOD(kNDArrayHandle);
+    value_.v_handle = reinterpret_cast<void*>(value->value);
+    return *this;
+  }
   MXNetRetValue& operator=(const PythonArg& value) {
     this->SwitchToPOD(kPyArg);
     value_.v_int64 = value.offset();
diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py
index d6d6a1f49e8e..83cf72d4c179 100644
--- a/python/mxnet/__init__.py
+++ b/python/mxnet/__init__.py
@@ -108,3 +108,4 @@
 
 from . import _api_internal
 from . import api
+from . import container
diff --git a/python/mxnet/_ffi/_ctypes/function.py b/python/mxnet/_ffi/_ctypes/function.py
index 0a005dd7b749..58823404909f 100644
--- a/python/mxnet/_ffi/_ctypes/function.py
+++ b/python/mxnet/_ffi/_ctypes/function.py
@@ -28,9 +28,10 @@
 from ..base import c_str
 from .types import MXNetValue, TypeCode
 from .types import RETURN_SWITCH
-from .object import ObjectBase
 from ..node_generic import convert_to_node
 from ..._ctypes.ndarray import NDArrayBase
+from .object import ObjectBase, _set_class_object
+from . import object as _object
 
 ObjectHandle = ctypes.c_void_p
 
@@ -118,8 +119,20 @@ def __call__(self, *args):
                 else RETURN_SWITCH[ret_tcode.value](ret_val, args))
 
 
-_CLASS_OBJECT = None
+def __init_handle_by_constructor__(fconstructor, args):
+    """Initialize handle by constructor"""
+    temp_args = []
+    values, tcodes, num_args = _make_mxnet_args(args, temp_args)
+    ret_val = MXNetValue()
+    ret_tcode = ctypes.c_int()
+    if _LIB.MXNetFuncCall(
+            fconstructor.handle, values, tcodes, ctypes.c_int(num_args),
+            ctypes.byref(ret_val), ctypes.byref(ret_tcode)) != 0:
+        raise get_last_ffi_error()
+    _ = temp_args
+    _ = args
+    assert ret_tcode.value == TypeCode.OBJECT_HANDLE
+    handle = ret_val.v_handle
+    return handle
 
-def _set_class_object(obj_class):
-    global _CLASS_OBJECT
-    _CLASS_OBJECT = obj_class
+_object.__init_by_constructor__ = __init_handle_by_constructor__
diff --git a/python/mxnet/_ffi/_ctypes/object.py b/python/mxnet/_ffi/_ctypes/object.py
index 85ab415692f6..241ac100de86 100644
--- a/python/mxnet/_ffi/_ctypes/object.py
+++ b/python/mxnet/_ffi/_ctypes/object.py
@@ -21,18 +21,35 @@
 """
 import ctypes
 from ...base import _LIB, check_call
-from . import function
 from .types import RETURN_SWITCH, TypeCode
 
 ObjectHandle = ctypes.c_void_p
+__init_by_constructor__ = None
+
+"""Maps object type to its constructor"""
+OBJECT_TYPE = {}
+
+_CLASS_OBJECT = None
+
+def _set_class_object(object_class):
+    global _CLASS_OBJECT
+    _CLASS_OBJECT = object_class
+
+def _register_object(index, cls):
+    """register object class"""
+    # if issubclass(cls, NDArrayBase):
+    #     _register_ndarray(index, cls)
+    #     return
+    OBJECT_TYPE[index] = cls
 
 
 def _return_object(x):
     handle = x.v_handle
     if not isinstance(handle, ObjectHandle):
         handle = ObjectHandle(handle)
-    # Does not support specific cpp node class for now
-    cls = function._CLASS_OBJECT
+    tindex = ctypes.c_uint()
+    check_call(_LIB.MXNetObjectGetTypeIndex(handle, ctypes.byref(tindex)))
+    cls = OBJECT_TYPE.get(tindex.value, _CLASS_OBJECT)
     # Avoid calling __init__ of cls, instead directly call __new__
     # This allows child class to implement their own __init__
     obj = cls.__new__(cls)
@@ -50,4 +67,26 @@ def __del__(self):
         if _LIB is not None:
             check_call(_LIB.MXNetObjectFree(self.handle))
 
-    # Does not support creation of cpp node class via python class
+    def __init_handle_by_constructor__(self, fconstructor, *args):
+        """Initialize the handle by calling constructor function.
+
+        Parameters
+        ----------
+        fconstructor : Function
+            Constructor function.
+
+        args: list of objects
+            The arguments to the constructor
+
+        Note
+        ----
+        We have a special calling convention to call constructor functions.
+        So the return handle is directly set into the Node object
+        instead of creating a new Node.
+        """
+        # assign handle first to avoid error raising
+        self.handle = None
+        handle = __init_by_constructor__(fconstructor, args)
+        if not isinstance(handle, ObjectHandle):
+            handle = ObjectHandle(handle)
+        self.handle = handle
diff --git a/python/mxnet/_ffi/_cython/base.pxi b/python/mxnet/_ffi/_cython/base.pxi
index bc2273bacd0d..84d02e0452c4 100644
--- a/python/mxnet/_ffi/_cython/base.pxi
+++ b/python/mxnet/_ffi/_cython/base.pxi
@@ -59,6 +59,8 @@ cdef extern from "mxnet/runtime/c_runtime_api.h":
                       MXNetValue* ret_val,
                       int* ret_type_code)
     int MXNetFuncFree(MXNetFunctionHandle func)
+    int MXNetObjectFree(ObjectHandle obj)
+    int MXNetObjectGetTypeIndex(ObjectHandle obj, unsigned* out_index)
 
 
 cdef inline py_str(const char* x):
diff --git a/python/mxnet/_ffi/_cython/core.pyx b/python/mxnet/_ffi/_cython/core.pyx
index 482f494b6e5e..0110eaaedc1b 100644
--- a/python/mxnet/_ffi/_cython/core.pyx
+++ b/python/mxnet/_ffi/_cython/core.pyx
@@ -20,4 +20,5 @@
 include "./base.pxi"
 include "./ndarray.pxi"
 include "./convert.pxi"
+include "./object.pxi"
 include "./function.pxi"
diff --git a/python/mxnet/_ffi/_cython/function.pxi b/python/mxnet/_ffi/_cython/function.pxi
index d4c629a618d5..1e6aa8625ec2 100644
--- a/python/mxnet/_ffi/_cython/function.pxi
+++ b/python/mxnet/_ffi/_cython/function.pxi
@@ -53,6 +53,9 @@ cdef inline int make_arg(object arg,
     elif arg is None:
         value[0].v_handle = NULL
         tcode[0] = kNull
+    elif isinstance(arg, ObjectBase):
+        value[0].v_handle = (<ObjectBase>arg).chandle
+        tcode[0] = kObjectHandle
     elif isinstance(arg, Number):
         value[0].v_float64 = arg
         tcode[0] = kFloat
@@ -77,6 +80,8 @@ cdef inline object make_ret(MXNetValue value, int tcode, tuple args):
         return args[value.v_int64]
     elif tcode == kNull:
         return None
+    elif tcode == kObjectHandle:
+        return make_ret_object(value.v_handle)
     elif tcode == kInt:
         return value.v_int64
     elif tcode == kFloat:
@@ -130,6 +135,19 @@ cdef inline int FuncCall(void* chandle,
     return 0
 
 
+cdef inline int ConstructorCall(void* constructor_handle,
+                                int type_code,
+                                tuple args,
+                                void** handle) except -1:
+    """Call contructor of a handle function"""
+    cdef MXNetValue ret_val
+    cdef int ret_tcode
+    FuncCall(constructor_handle, args, &ret_val, &ret_tcode)
+    assert ret_tcode == type_code
+    handle[0] = ret_val.v_handle
+    return 0
+
+
 cdef class FunctionBase:
     cdef MXNetFunctionHandle chandle
     cdef int is_global
@@ -169,3 +187,11 @@ cdef class FunctionBase:
         cdef int ret_tcode
         FuncCall(self.chandle, args, &ret_val, &ret_tcode)
         return make_ret(ret_val, ret_tcode, args)
+
+
+_CLASS_OBJECT = None
+
+
+def _set_class_object(obj_class):
+    global _CLASS_OBJECT
+    _CLASS_OBJECT = obj_class
diff --git a/python/mxnet/_ffi/_cython/object.pxi b/python/mxnet/_ffi/_cython/object.pxi
new file mode 100644
index 000000000000..4f31f2ad5aaa
--- /dev/null
+++ b/python/mxnet/_ffi/_cython/object.pxi
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Maps object type to its constructor
+Acknowledgement: This file originates from incubator-tvm
+"""
+cdef list OBJECT_TYPE = []
+
+def _register_object(int index, object cls):
+    """register object class"""
+    global OBJECT_TYPE
+    while len(OBJECT_TYPE) <= index:
+        OBJECT_TYPE.append(None)
+    OBJECT_TYPE[index] = cls
+
+
+cdef inline object make_ret_object(void* chandle):
+    global OBJECT_TYPE
+    global _CLASS_OBJECT
+    cdef unsigned tindex
+    cdef object cls
+    object_type = OBJECT_TYPE
+    CALL(MXNetObjectGetTypeIndex(chandle, &tindex))
+    if tindex < len(OBJECT_TYPE):
+        cls = OBJECT_TYPE[tindex]
+        if cls is not None:
+            obj = cls.__new__(cls)
+        else:
+            obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)
+    else:
+        obj = _CLASS_OBJECT.__new__(_CLASS_OBJECT)
+    (<ObjectBase>obj).chandle = chandle
+    return obj
+
+
+cdef class ObjectBase:
+    cdef void* chandle
+
+    cdef inline _set_handle(self, handle):
+        cdef unsigned long long ptr
+        if handle is None:
+            self.chandle = NULL
+        else:
+            ptr = handle.value
+            self.chandle = <void*>(ptr)
+
+    property handle:
+        def __get__(self):
+            if self.chandle == NULL:
+                return None
+            else:
+                return ctypes_handle(self.chandle)
+
+        def __set__(self, value):
+            self._set_handle(value)
+
+    def __dealloc__(self):
+        CALL(MXNetObjectFree(self.chandle))
+
+    def __init_handle_by_constructor__(self, fconstructor, *args):
+        """Initialize the handle by calling constructor function.
+
+        Parameters
+        ----------
+        fconstructor : Function
+            Constructor function.
+
+        args: list of objects
+            The arguments to the constructor
+
+        Note
+        ----
+        We have a special calling convention to call constructor functions.
+        So the return handle is directly set into the Node object
+        instead of creating a new Node.
+        """
+        # avoid error raised during construction.
+        self.chandle = NULL
+        cdef void* chandle
+        ConstructorCall(
+            (<FunctionBase>fconstructor).chandle,
+            kObjectHandle, args, &chandle)
+        self.chandle = chandle
diff --git a/python/mxnet/_ffi/object.py b/python/mxnet/_ffi/object.py
index e0a4aa600f25..5435a70928a6 100644
--- a/python/mxnet/_ffi/object.py
+++ b/python/mxnet/_ffi/object.py
@@ -17,10 +17,107 @@
 # pylint: disable=invalid-name
 """Runtime Object API
 Acknowledgement: This file originates from incubator-tvm"""
-from ._ctypes.function import _set_class_object
-from ._ctypes.object import ObjectBase as _ObjectBase
+import os
+import ctypes
+from ..base import _LIB, check_call, c_str
+
+try:
+    if int(os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0:
+        from ._ctypes.function import _set_class_object
+        from ._ctypes.object import ObjectBase as _ObjectBase
+        from ._ctypes.object import _register_object
+    else:
+        from ._cy3.core import _set_class_object
+        from ._cy3.core import ObjectBase as _ObjectBase
+        from ._cy3.core import _register_object
+except ImportError:
+    if int(os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0:
+        raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1")
+    from ._ctypes.function import _set_class_object
+    from ._ctypes.object import ObjectBase as _ObjectBase
+    from ._ctypes.object import _register_object
 
 class Object(_ObjectBase):
     """Base class for all mxnet's runtime objects."""
 
+
+def register_object(type_key=None):
+    """register object type.
+
+    Parameters
+    ----------
+    type_key : str or cls
+        The type key of the node
+
+    Examples
+    --------
+    The following code registers MyObject
+    using type key "test.MyObject"
+
+    .. code-block:: python
+
+      @register_object("test.MyObject")
+      class MyObject(Object):
+          pass
+    """
+    object_name = type_key if isinstance(type_key, str) else type_key.__name__
+
+    def register(cls):
+        """internal register function"""
+        if hasattr(cls, "_type_index"):
+            tindex = cls._type_index
+        else:
+            tidx = ctypes.c_uint()
+            check_call(_LIB.MXNetObjectTypeKey2Index(
+                c_str(object_name), ctypes.byref(tidx)))
+            tindex = tidx.value
+        _register_object(tindex, cls)
+        return cls
+
+    if isinstance(type_key, str):
+        return register
+
+    return register(type_key)
+
+
+def getitem_helper(obj, elem_getter, length, idx):
+    """Helper function to implement a pythonic getitem function.
+
+    Parameters
+    ----------
+    obj: object
+        The original object
+
+    elem_getter : function
+        A simple function that takes index and return a single element.
+
+    length : int
+        The size of the array
+
+    idx : int or slice
+        The argument passed to getitem
+
+    Returns
+    -------
+    result : object
+        The result of getitem
+    """
+    if isinstance(idx, slice):
+        start = idx.start if idx.start is not None else 0
+        stop = idx.stop if idx.stop is not None else length
+        step = idx.step if idx.step is not None else 1
+        if start < 0:
+            start += length
+        if stop < 0:
+            stop += length
+        return [elem_getter(obj, i) for i in range(start, stop, step)]
+
+    if idx < -length or idx >= length:
+        raise IndexError("Index out of range. size: {}, got index {}"
+                         .format(length, idx))
+    if idx < 0:
+        idx += length
+    return elem_getter(obj, idx)
+
+
 _set_class_object(Object)
diff --git a/python/mxnet/container.py b/python/mxnet/container.py
new file mode 100644
index 000000000000..f0760b3b1d21
--- /dev/null
+++ b/python/mxnet/container.py
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=undefined-variable
+"""
+Container data structures.
+Acknowledgement: This file originates from incubator-tvm
+"""
+from ._ffi.object import Object, register_object, getitem_helper
+from ._ffi.function import _init_api
+
+@register_object("MXNet.ADT")
+class ADT(Object):
+    """Algebatic data type(ADT) object.
+
+    Parameters
+    ----------
+    tag : int
+        The tag of ADT.
+
+    fields : list[Object] or tuple[Object]
+        The source tuple.
+    """
+    def __init__(self, tag, fields):
+        for f in fields:
+            assert isinstance(f, (Object)), "Expect object" \
+            ", but received : {0}".format(type(f))
+        self.__init_handle_by_constructor__(_ADT, tag, *fields)
+
+    @property
+    def tag(self):
+        return _GetADTTag(self)
+
+    def __getitem__(self, idx):
+        return getitem_helper(
+            self, _GetADTFields, len(self), idx)
+
+    def __len__(self):
+        return _GetADTSize(self)
+
+_init_api("mxnet.container")
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index ae53645bb4e0..db2701e47ab3 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -990,7 +990,7 @@ def add(x1, x2, out=None, **kwargs):
         * If both inputs are of integer types (including boolean), not supported yet.
     """
     if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
-        _np.add(x1, x2, out=out)
+        return _np.add(x1, x2, out=out)
     return _api_internal.add(x1, x2, out)
 
 
@@ -3698,21 +3698,9 @@ def split(ary, indices_or_sections, axis=0):
         If `indices_or_sections` is given as an integer, but
         a split does not result in equal division.
     """
-    axis_size = ary.shape[axis]
-    if isinstance(indices_or_sections, integer_types):
-        sections = indices_or_sections
-        if axis_size % sections:
-            raise ValueError('array split does not result in an equal division')
-        section_size = int(axis_size / sections)
-        indices = [i * section_size for i in range(sections)]
-    elif isinstance(indices_or_sections, (list, set, tuple)):
-        indices = [0] + list(indices_or_sections)
-    else:
-        raise ValueError('indices_or_sections must be either int, or tuple / list / set of ints')
-    ret = _npi.split(ary, indices, axis, False)
-    assert isinstance(ret, list), 'Output of split should be list,' \
-                                  ' got a return type {}'.format(type(ret))
-    return ret
+    if isinstance(indices_or_sections, set):
+        indices_or_sections = list(indices_or_sections)
+    return list(_api_internal.split(ary, indices_or_sections, axis))
 # pylint: enable=redefined-outer-name
 
 
diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index 0acedf4bbab4..fdcbdac2247a 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -19,6 +19,7 @@
 
 from . import _op as _mx_nd_np
 from . import _internal as _npi
+from . import _api_internal
 
 __all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
            'eigvals', 'eig', 'eigvalsh', 'eigh']
@@ -329,7 +330,7 @@ def svd(a):
     >>> (ret - a < -1e-3).sum()
     array(0.)
     """
-    return tuple(_npi.svd(a))
+    return tuple(_api_internal.svd(a))
 
 
 def cholesky(a):
diff --git a/src/api/operator/numpy/linalg/np_gesvd.cc b/src/api/operator/numpy/linalg/np_gesvd.cc
new file mode 100644
index 000000000000..a4517849cbaf
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_gesvd.cc
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_gesvd.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_gesvd.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.svd")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = Op::Get("_npi_svd");
+  attrs.op = op;
+  // inputs
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+  int num_inputs = 1;
+  // outputs
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1]),
+                 NDArrayHandle(ndoutputs[2])});
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index 080cca867ecb..cc268c202c9b 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -46,4 +46,43 @@ MXNET_REGISTER_API("_npi.expand_dims")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.split")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_split");
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = args[2].operator int();
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+    CHECK_EQ(inputs[0]->shape()[param.axis] % param.sections, 0)
+      << "ValueError: array split does not result in an equal division";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
 }  // namespace mxnet
diff --git a/src/api/operator/numpy/np_tensordot_op.cc b/src/api/operator/numpy/np_tensordot_op.cc
index b163757f85b1..eef58b5b3389 100644
--- a/src/api/operator/numpy/np_tensordot_op.cc
+++ b/src/api/operator/numpy/np_tensordot_op.cc
@@ -39,8 +39,9 @@ inline static void _npi_tensordot_int_axes(runtime::MXNetArgs args,
   attrs.parsed = param;
   SetAttrDict<op::TensordotIntAxesParam>(&attrs);
   int num_outputs = 0;
+  int num_inputs = 2;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 2, inputs, &num_outputs, nullptr);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
 }
 
@@ -52,9 +53,11 @@ inline static void _npi_tensordot(runtime::MXNetArgs args,
   nnvm::NodeAttrs attrs;
   ADT adt = Downcast<ADT, ObjectRef>(args[2].operator ObjectRef());
   if (const IntegerObj* lop = adt[0].as<IntegerObj>()) {
+    // axes is a tuple of int, like axes=(0, 1)
     param.a_axes_summed = Tuple<int>(1, lop->value);
     param.b_axes_summed = Tuple<int>(1, Downcast<Integer, ObjectRef>(adt[1])->value);
   } else {
+    // axes is a tuple of tuples of int, like axes=((0, 1), (1, 0))
     param.a_axes_summed = Tuple<int>(adt[0]);
     param.b_axes_summed = Tuple<int>(adt[1]);
   }
@@ -62,8 +65,9 @@ inline static void _npi_tensordot(runtime::MXNetArgs args,
   attrs.parsed = std::move(param);
   SetAttrDict<op::TensordotParam>(&attrs);
   int num_outputs = 0;
+  int num_inputs = 2;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 2, inputs, &num_outputs, nullptr);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
 }
 
diff --git a/src/api/operator/op_utils.cc b/src/api/operator/op_utils.cc
index bb54662e7a62..1cf813eb8688 100644
--- a/src/api/operator/op_utils.cc
+++ b/src/api/operator/op_utils.cc
@@ -28,7 +28,7 @@
 
 namespace mxnet {
 
-std::string String2MXNetTypeWithBool(int dtype) {
+std::string MXNetTypeWithBool2String(int dtype) {
   switch (dtype) {
     case mshadow::kFloat32:
       return "float32";
diff --git a/src/api/operator/op_utils.h b/src/api/operator/op_utils.h
index f41680df6fd6..285919cd14c4 100644
--- a/src/api/operator/op_utils.h
+++ b/src/api/operator/op_utils.h
@@ -28,7 +28,7 @@
 
 namespace mxnet {
 
-std::string String2MXNetTypeWithBool(int dtype);
+std::string MXNetTypeWithBool2String(int dtype);
 std::string MXNetPercentileType2String(int interpolation);
 
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_cumsum-inl.h b/src/operator/numpy/np_cumsum-inl.h
index dfba843d7be6..9bfcfc132297 100644
--- a/src/operator/numpy/np_cumsum-inl.h
+++ b/src/operator/numpy/np_cumsum-inl.h
@@ -64,7 +64,7 @@ struct CumsumParam : public dmlc::Parameter<CumsumParam> {
     dtype_s << dtype;
     (*dict)["axis"] = axis_s.str();
     if (dtype.has_value()) {
-      (*dict)["dtype"] = String2MXNetTypeWithBool(dtype.value());
+      (*dict)["dtype"] = MXNetTypeWithBool2String(dtype.value());
     } else {
       (*dict)["dtype"] = dtype_s.str();
     }
diff --git a/src/operator/numpy/np_cumsum.cc b/src/operator/numpy/np_cumsum.cc
index ea0f9b6b11bc..594f7b796347 100644
--- a/src/operator/numpy/np_cumsum.cc
+++ b/src/operator/numpy/np_cumsum.cc
@@ -66,6 +66,7 @@ inline bool CumsumType(const nnvm::NodeAttrs& attrs,
 DMLC_REGISTER_PARAMETER(CumsumParam);
 
 NNVM_REGISTER_OP(_npi_cumsum)
+.add_alias("cumsum")
 .describe(R"code(Return the cumulative sum of the elements along a given axis.)code" ADD_FILELINE)
 .set_attr_parser(ParamParser<CumsumParam>)
 .set_num_inputs(1)
diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h
index f6610a980f6a..fb739c690607 100644
--- a/src/operator/tensor/init_op.h
+++ b/src/operator/tensor/init_op.h
@@ -39,6 +39,7 @@
 #include "../elemwise_op_common.h"
 #include "../mxnet_op.h"
 #include "../mshadow_op.h"
+#include "../../api/operator/op_utils.h"
 
 
 namespace mxnet {
@@ -61,11 +62,10 @@ struct InitOpParam : public dmlc::Parameter<InitOpParam> {
     .describe("Target data type.");
   }
   void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
-    std::ostringstream shape_s, dtype_s;
+    std::ostringstream shape_s;
     shape_s << shape;
-    dtype_s << dtype;
     (*dict)["shape"] = shape_s.str();
-    (*dict)["dtype"] = dtype_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
     // We do not set ctx, because ctx has been set in dict instead of InitOpParam.
     // Setting ctx here results in an error.
   }
diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h
index 670104bcfdb0..6efde79f202b 100644
--- a/src/operator/tensor/matrix_op-inl.h
+++ b/src/operator/tensor/matrix_op-inl.h
@@ -2729,6 +2729,17 @@ struct SplitParam : public dmlc::Parameter<SplitParam> {
     DMLC_DECLARE_FIELD(sections).set_default(0)
     .describe("Number of sections if equally splitted. Default to 0 which means split by indices.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream indices_s, axis_s, squeeze_axis_s, sections_s;
+    indices_s << indices;
+    axis_s << axis;
+    squeeze_axis_s << squeeze_axis;
+    sections_s << sections;
+    (*dict)["indices"] = indices_s.str();
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["squeeze_axis"] = squeeze_axis_s.str();
+    (*dict)["sections"] = sections_s.str();
+  }
 };  // struct SplitParam
 
 inline mxnet::TShape GetSplitIndices(const mxnet::TShape& ishape, int axis, int sections) {
diff --git a/src/runtime/container.cc b/src/runtime/container.cc
new file mode 100644
index 000000000000..f47e2290789f
--- /dev/null
+++ b/src/runtime/container.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/container.cc
+ * \brief Implementations of common plain old data (POD) containers.
+ */
+// Acknowledgement: This file originates from incubator-tvm
+#include <mxnet/runtime/container.h>
+#include <mxnet/runtime/memory.h>
+#include <mxnet/runtime/object.h>
+#include <mxnet/runtime/registry.h>
+
+namespace mxnet {
+namespace runtime {
+
+MXNET_REGISTER_GLOBAL("container._GetADTTag")
+.set_body([](MXNetArgs args, MXNetRetValue* rv) {
+  ObjectRef obj = args[0];
+  const auto& adt = Downcast<ADT>(obj);
+  *rv = static_cast<int64_t>(adt.tag());
+});
+
+MXNET_REGISTER_GLOBAL("container._GetADTSize")
+.set_body([](MXNetArgs args, MXNetRetValue* rv) {
+  ObjectRef obj = args[0];
+  const auto& adt = Downcast<ADT>(obj);
+  *rv = static_cast<int64_t>(adt.size());
+});
+
+
+MXNET_REGISTER_GLOBAL("container._GetADTFields")
+.set_body([](MXNetArgs args, MXNetRetValue* rv) {
+  ObjectRef obj = args[0];
+  int idx = args[1];
+  const auto& adt = Downcast<ADT>(obj);
+  CHECK_LT(idx, adt.size());
+  *rv = adt[idx];
+});
+
+MXNET_REGISTER_GLOBAL("container._ADT")
+.set_body([](MXNetArgs args, MXNetRetValue* rv) {
+  int itag = args[0];
+  size_t tag = static_cast<size_t>(itag);
+  std::vector<ObjectRef> fields;
+  for (int i = 1; i < args.size(); i++) {
+    fields.push_back(args[i]);
+  }
+  *rv = ADT(tag, fields);
+});
+
+MXNET_REGISTER_OBJECT_TYPE(ADTObj);
+
+}  // namespace runtime
+
+}  // namespace mxnet
diff --git a/src/runtime/ndarray_handle.cc b/src/runtime/ndarray_handle.cc
new file mode 100644
index 000000000000..5afb1984740b
--- /dev/null
+++ b/src/runtime/ndarray_handle.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/api/ndarary_handle.cc
+ * \brief Implementations of NDArrayHandle
+ */
+#include <mxnet/runtime/ndarray_handle.h>
+#include <mxnet/runtime/object.h>
+#include <mxnet/runtime/registry.h>
+#include <mxnet/runtime/packed_func.h>
+
+namespace mxnet {
+namespace runtime {
+
+MXNET_REGISTER_GLOBAL("ndarray_handle._GetNDArrayHandleValue")
+.set_body([](MXNetArgs args, MXNetRetValue* rv) {
+  ObjectRef obj = args[0];
+  const auto& handle = Downcast<NDArrayHandle>(obj);
+  *rv = handle->value;
+});
+
+MXNET_REGISTER_OBJECT_TYPE(NDArrayHandleObj);
+
+}  // namespace runtime
+}  // namespace mxnet
diff --git a/src/runtime/object.cc b/src/runtime/object.cc
index 76d8b9776d8a..ee7ed74ecb88 100644
--- a/src/runtime/object.cc
+++ b/src/runtime/object.cc
@@ -213,3 +213,17 @@ int MXNetObjectFree(MXNetObjectHandle obj) {
   mxnet::runtime::ObjectInternal::ObjectFree(obj);
   API_END();
 }
+
+int MXNetObjectGetTypeIndex(MXNetObjectHandle obj, unsigned* out_tindex) {
+  API_BEGIN();
+  CHECK(obj != nullptr);
+  out_tindex[0] = static_cast<mxnet::runtime::Object*>(obj)->type_index();
+  API_END();
+}
+
+int MXNetObjectTypeKey2Index(const char* type_key, unsigned* out_tindex) {
+  API_BEGIN();
+  out_tindex[0] = mxnet::runtime::ObjectInternal::ObjectTypeKey2Index(
+      type_key);
+  API_END();
+}
diff --git a/src/runtime/object_internal.h b/src/runtime/object_internal.h
index 002468456741..7252bace9491 100644
--- a/src/runtime/object_internal.h
+++ b/src/runtime/object_internal.h
@@ -46,6 +46,15 @@ class ObjectInternal {
       static_cast<Object*>(obj)->DecRef();
     }
   }
+
+  /*!
+   * \brief Expose TypeKey2Index
+   * \param type_key The original type key.
+   * \return the corresponding index.
+   */
+  static uint32_t ObjectTypeKey2Index(const std::string& type_key) {
+    return Object::TypeKey2Index(type_key);
+  }
 };
 
 }  // namespace runtime

From 7ed6bf7a3d5a71b36c060ea751ccf27b702333c4 Mon Sep 17 00:00:00 2001
From: alicia <32725332+Alicia1529@users.noreply.github.com>
Date: Wed, 18 Mar 2020 15:14:21 +0800
Subject: [PATCH 10/44] add ffi for full_like, binary (#17811)

---
 benchmark/python/ffi/benchmark_ffi.py         | 16 ++++++
 python/mxnet/ndarray/numpy/_op.py             | 57 +++++++++++--------
 .../numpy/np_broadcast_reduce_op_value.cc     |  3 +-
 src/api/operator/numpy/np_diff_op.cc          |  3 +-
 .../numpy/np_elemwise_broadcast_op.cc         | 52 +++++++++++++++++
 src/api/operator/numpy/np_init_op.cc          | 33 +++++++++++
 src/api/operator/numpy/np_matrix_op.cc        |  3 +-
 src/api/operator/numpy/np_nonzero_op.cc       | 45 +++++++++++++++
 src/api/operator/numpy/np_tril_op.cc          |  3 +-
 src/operator/tensor/init_op.h                 | 11 ++++
 10 files changed, 199 insertions(+), 27 deletions(-)
 create mode 100644 src/api/operator/numpy/np_nonzero_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index c420ee9cdad3..3e372a659c4c 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -59,6 +59,22 @@ def prepare_workloads():
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
+    OpArgMngr.add_workload("subtract", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("multiply", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("mod", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("remainder", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("divide", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("true_divide", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("power", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("lcm", pool['2x2'].astype('int32'), pool['2x2'].astype('int32'))
+    OpArgMngr.add_workload("diff", pool['2x2'], n=1, axis=-1)
+    OpArgMngr.add_workload("nonzero", pool['2x2'])
+    OpArgMngr.add_workload("tril", pool['2x2'], k=0)
+    OpArgMngr.add_workload("expand_dims", pool['2x2'], axis=0)
+    OpArgMngr.add_workload("broadcast_to", pool['2x2'], (2, 2, 2))
+    OpArgMngr.add_workload("full_like", pool['2x2'], 2)
+    OpArgMngr.add_workload("zeros_like", pool['2x2'])
+    OpArgMngr.add_workload("ones_like", pool['2x2'])
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
 
 
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index db2701e47ab3..9cd49774f595 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -212,9 +212,7 @@ def zeros_like(a, dtype=None, order='C', ctx=None, out=None):
     """
     if order != 'C':
         raise NotImplementedError
-    if ctx is None:
-        ctx = current_context()
-    return _npi.full_like(a, fill_value=0, dtype=dtype, ctx=ctx, out=out)
+    return full_like(a, 0, dtype=dtype, order=order, ctx=ctx, out=out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -270,11 +268,7 @@ def ones_like(a, dtype=None, order='C', ctx=None, out=None):
     >>> np.ones_like(y)
     array([1., 1., 1.], dtype=float64)
     """
-    if order != 'C':
-        raise NotImplementedError
-    if ctx is None:
-        ctx = current_context()
-    return _npi.full_like(a, fill_value=1, dtype=dtype, ctx=ctx, out=out)
+    return full_like(a, 1, dtype=dtype, order=order, ctx=ctx, out=out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -433,11 +427,15 @@ def full_like(a, fill_value, dtype=None, order='C', ctx=None, out=None): # pylin
     """
     if order != 'C':
         raise NotImplementedError
-    if ctx is None:
-        ctx = current_context()
     if isinstance(fill_value, bool):
         fill_value = int(fill_value)
-    return _npi.full_like(a, fill_value=fill_value, dtype=dtype, ctx=ctx, out=out)
+    if ctx is None:
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.full_like(a, fill_value, dtype, ctx, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1025,8 +1023,9 @@ def subtract(x1, x2, out=None, **kwargs):
         * If only one of the inputs is floating number type, the result is that type.
         * If both inputs are of integer types (including boolean), not supported yet.
     """
-    return _ufunc_helper(x1, x2, _npi.subtract, _np.subtract, _npi.subtract_scalar,
-                         _npi.rsubtract_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.subtract(x1, x2, out=out)
+    return _api_internal.subtract(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1060,7 +1059,9 @@ def multiply(x1, x2, out=None, **kwargs):
         * If only one of the inputs is floating number type, the result is that type.
         * If both inputs are of integer types (including boolean), not supported yet.
     """
-    return _ufunc_helper(x1, x2, _npi.multiply, _np.multiply, _npi.multiply_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.multiply(x1, x2, out=out)
+    return _api_internal.multiply(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1095,8 +1096,9 @@ def divide(x1, x2, out=None, **kwargs):
         * If only one of the inputs is floating number type, the result is that type.
         * If both inputs are of integer types (including boolean), the output is of float32 type.
     """
-    return _ufunc_helper(x1, x2, _npi.true_divide, _np.divide, _npi.true_divide_scalar,
-                         _npi.rtrue_divide_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.divide(x1, x2, out=out)
+    return _api_internal.true_divide(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1133,8 +1135,9 @@ def true_divide(x1, x2, out=None):
         * If only one of the inputs is floating number type, the result is that type.
         * If both inputs are of integer types (including boolean), the output is of float32 type.
     """
-    return _ufunc_helper(x1, x2, _npi.true_divide, _np.divide, _npi.true_divide_scalar,
-                         _npi.rtrue_divide_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.true_divide(x1, x2, out=out)
+    return _api_internal.true_divide(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1161,7 +1164,9 @@ def mod(x1, x2, out=None, **kwargs):
     out : ndarray or scalar
         This is a scalar if both x1 and x2 are scalars.
     """
-    return _ufunc_helper(x1, x2, _npi.mod, _np.mod, _npi.mod_scalar, _npi.rmod_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.mod(x1, x2, out=out)
+    return _api_internal.mod(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1349,7 +1354,9 @@ def remainder(x1, x2, out=None):
     out : ndarray or scalar
         This is a scalar if both x1 and x2 are scalars.
     """
-    return _ufunc_helper(x1, x2, _npi.mod, _np.mod, _npi.mod_scalar, _npi.rmod_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        _np.mod(x1, x2, out=out)
+    return _api_internal.mod(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1377,7 +1384,9 @@ def power(x1, x2, out=None, **kwargs):
         The bases in x1 raised to the exponents in x2.
         This is a scalar if both x1 and x2 are scalars.
     """
-    return _ufunc_helper(x1, x2, _npi.power, _np.power, _npi.power_scalar, _npi.rpower_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.power(x1, x2, out=out)
+    return _api_internal.power(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1976,7 +1985,9 @@ def lcm(x1, x2, out=None, **kwargs):
     >>> np.lcm(np.arange(6, dtype=int), 20)
     array([ 0, 20, 20, 60, 20, 20], dtype=int64)
     """
-    return _ufunc_helper(x1, x2, _npi.lcm, _np.lcm, _npi.lcm_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.lcm(x1, x2, out=out)
+    return _api_internal.lcm(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6658,7 +6669,7 @@ def nonzero(a):
     >>> (a > 3).nonzero()
     (array([1, 1, 1, 2, 2, 2], dtype=int64), array([0, 1, 2, 0, 1, 2], dtype=int64))
     """
-    out = _npi.nonzero(a).transpose()
+    out = _api_internal.nonzero(a).transpose()
     return tuple([out[i] for i in range(len(out))])
 
 
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
index 2322860aa609..224451c70570 100644
--- a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
@@ -46,7 +46,8 @@ MXNET_REGISTER_API("_npi.broadcast_to")
 
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  int num_inputs = 1;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = ndoutputs[0];
 });
 
diff --git a/src/api/operator/numpy/np_diff_op.cc b/src/api/operator/numpy/np_diff_op.cc
index dec73b8496a7..7be5b804eade 100644
--- a/src/api/operator/numpy/np_diff_op.cc
+++ b/src/api/operator/numpy/np_diff_op.cc
@@ -43,7 +43,8 @@ MXNET_REGISTER_API("_npi.diff")
 
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  int num_inputs = 1;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = ndoutputs[0];
 });
 
diff --git a/src/api/operator/numpy/np_elemwise_broadcast_op.cc b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
index e724a7c58bd3..7a9eb0139439 100644
--- a/src/api/operator/numpy/np_elemwise_broadcast_op.cc
+++ b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
@@ -36,4 +36,56 @@ MXNET_REGISTER_API("_npi.add")
   UFuncHelper(args, ret, op, op_scalar, nullptr);
 });
 
+MXNET_REGISTER_API("_npi.subtract")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_subtract");
+  const nnvm::Op* op_scalar = Op::Get("_npi_subtract_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rsubtract_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.multiply")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_multiply");
+  const nnvm::Op* op_scalar = Op::Get("_npi_multiply_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.true_divide")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_true_divide");
+  const nnvm::Op* op_scalar = Op::Get("_npi_true_divide_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rtrue_divide_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.mod")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_mod");
+  const nnvm::Op* op_scalar = Op::Get("_npi_mod_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rmod_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.power")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_power");
+  const nnvm::Op* op_scalar = Op::Get("_npi_power_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rpower_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.lcm")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_lcm");
+  const nnvm::Op* op_scalar = Op::Get("_npi_lcm_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
 }  // namespace mxnet
diff --git a/src/api/operator/numpy/np_init_op.cc b/src/api/operator/numpy/np_init_op.cc
index c65f90c841f4..4f7c6e497616 100644
--- a/src/api/operator/numpy/np_init_op.cc
+++ b/src/api/operator/numpy/np_init_op.cc
@@ -21,6 +21,7 @@
  * \file np_init_op.cc
  * \brief Implementation of the API of functions in src/operator/numpy/np_init_op.cc
  */
+#include <dmlc/optional.h>
 #include <mxnet/api_registry.h>
 #include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
@@ -55,4 +56,36 @@ MXNET_REGISTER_API("_npi.zeros")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.full_like")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_full_like");
+  nnvm::NodeAttrs attrs;
+  op::FullLikeOpParam param;
+  param.fill_value = args[1].operator double();
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::nullopt;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  SetAttrDict<op::FullLikeOpParam>(&attrs);
+  NDArray* out = args[4].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(4);
+  } else {
+    *ret = ndoutputs[0];
+  }
+  *ret = ndoutputs[0];
+});
+
 }  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index cc268c202c9b..b4bb583c0511 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -42,7 +42,8 @@ MXNET_REGISTER_API("_npi.expand_dims")
 
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  int num_inputs = 1;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = ndoutputs[0];
 });
 
diff --git a/src/api/operator/numpy/np_nonzero_op.cc b/src/api/operator/numpy/np_nonzero_op.cc
new file mode 100644
index 000000000000..85510633c054
--- /dev/null
+++ b/src/api/operator/numpy/np_nonzero_op.cc
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_nonzero_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_nonzero_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.nonzero")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_nonzero");
+  nnvm::NodeAttrs attrs;
+
+  attrs.op = op;
+
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_tril_op.cc b/src/api/operator/numpy/np_tril_op.cc
index 105ff58bf559..1acb1b8e4b10 100644
--- a/src/api/operator/numpy/np_tril_op.cc
+++ b/src/api/operator/numpy/np_tril_op.cc
@@ -42,7 +42,8 @@ MXNET_REGISTER_API("_npi.tril")
 
   int num_outputs = 0;
   NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
-  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  int num_inputs = 1;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
   *ret = ndoutputs[0];
 });
 
diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h
index fb739c690607..ed0569c799ce 100644
--- a/src/operator/tensor/init_op.h
+++ b/src/operator/tensor/init_op.h
@@ -105,6 +105,17 @@ struct FullLikeOpParam : public dmlc::Parameter<FullLikeOpParam> {
       MXNET_ADD_ALL_TYPES_WITH_BOOL
       .describe("Target data type.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream fill_value_s, dtype_s;
+    fill_value_s << fill_value;
+    dtype_s << dtype;
+    (*dict)["fill_value"] = fill_value_s.str();
+    if (dtype.has_value()) {
+      (*dict)["dtype"] = MXNetTypeWithBool2String(dtype.value());
+    } else {
+      (*dict)["dtype"] = dtype_s.str();
+    }
+  }
 };
 
 /*! \brief Infer type of FullLikeOpCompute*/

From a81d5f43ce7c6f049d8322fac9ac66fe1b967300 Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Wed, 18 Mar 2020 00:36:36 +0800
Subject: [PATCH 11/44] * impl - FFI for np_where_op (#17817)

* impl - FFI for np_may_share_memory

* impl - FFI benchmark

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py  |   2 +
 python/mxnet/ndarray/numpy/_op.py      |  15 +---
 src/api/operator/numpy/np_memory_op.cc |  43 +++++++++++
 src/api/operator/numpy/np_where_op.cc  | 101 +++++++++++++++++++++++++
 src/operator/numpy/np_where_op-inl.h   |  12 +++
 5 files changed, 161 insertions(+), 12 deletions(-)
 create mode 100644 src/api/operator/numpy/np_memory_op.cc
 create mode 100644 src/api/operator/numpy/np_where_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 3e372a659c4c..2b9d8dfa58fe 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -76,6 +76,8 @@ def prepare_workloads():
     OpArgMngr.add_workload("zeros_like", pool['2x2'])
     OpArgMngr.add_workload("ones_like", pool['2x2'])
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
+    OpArgMngr.add_workload("where", pool['2x3'], pool['2x3'], pool['2x1'])
+    OpArgMngr.add_workload("may_share_memory", pool['2x3'][:0], pool['2x3'][:1])
 
 
 def benchmark_helper(f, *args, **kwargs):
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 9cd49774f595..e235c54b8711 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -6854,7 +6854,7 @@ def shares_memory(a, b, max_work=None):
     - Does not support `max_work`, it is a dummy argument
     - Actually it is same as `may_share_memory` in MXNet DeepNumPy
     """
-    return _npi.share_memory(a, b).item()
+    return _api_internal.share_memory(a, b).item()
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6895,7 +6895,7 @@ def may_share_memory(a, b, max_work=None):
     - Does not support `max_work`, it is a dummy argument
     - Actually it is same as `shares_memory` in MXNet DeepNumPy
     """
-    return _npi.share_memory(a, b).item()
+    return _api_internal.share_memory(a, b).item()
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7464,16 +7464,7 @@ def where(condition, x=None, y=None):  # pylint: disable=too-many-return-stateme
             else:
                 return y
         else:
-            if isinstance(x, numeric_types) and isinstance(y, numeric_types):
-                return _npi.where_scalar2(condition, float(x), float(y), out=None)
-            elif isinstance(x, NDArray) and isinstance(y, NDArray):
-                return _npi.where(condition, x, y, out=None)
-            elif isinstance(y, NDArray):
-                return _npi.where_lscalar(condition, y, float(x), out=None)
-            elif isinstance(x, NDArray):
-                return _npi.where_rscalar(condition, x, float(y), out=None)
-            else:
-                raise TypeError('type {0} and {1} not supported'.format(str(type(x)), str(type(y))))
+            return _api_internal.where(condition, x, y)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/src/api/operator/numpy/np_memory_op.cc b/src/api/operator/numpy/np_memory_op.cc
new file mode 100644
index 000000000000..33e5d4cfb7d8
--- /dev/null
+++ b/src/api/operator/numpy/np_memory_op.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_memory_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_memory_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.share_memory")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_share_memory");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_where_op.cc b/src/api/operator/numpy/np_where_op.cc
new file mode 100644
index 000000000000..a2ed14b042d7
--- /dev/null
+++ b/src/api/operator/numpy/np_where_op.cc
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_where_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_where_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_where_op-inl.h"
+
+namespace mxnet {
+
+inline static bool isScalar(const runtime::MXNetArgValue& arg) {
+  return arg.type_code() == kDLInt ||
+         arg.type_code() == kDLUInt ||
+         arg.type_code() == kDLFloat;
+}
+
+inline static void _npi_where(runtime::MXNetArgs args,
+                              runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_where");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 3;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(),
+                       args[1].operator mxnet::NDArray*(),
+                       args[2].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+inline static void _npi_where_scalar1(runtime::MXNetArgs args,
+                                      runtime::MXNetRetValue* ret,
+                                      bool isl) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = isl ? Op::Get("_npi_where_lscalar") : Op::Get("_npi_where_rscalar");
+  op::NumpyWhereScalarParam param;
+  param.scalar = isl ? args[1].operator double() : args[2].operator double();
+  attrs.op = op;
+  attrs.parsed = param;
+  SetAttrDict<op::NumpyWhereScalarParam>(&attrs);
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] =
+    {args[0].operator mxnet::NDArray*(),
+     isl ? args[2].operator mxnet::NDArray*() : args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+inline static void _npi_where_scalar2(runtime::MXNetArgs args,
+                                      runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_where_scalar2");
+  op::NumpyWhereScalar2Param param;
+  nnvm::NodeAttrs attrs;
+  param.x = args[1].operator double();
+  param.x = args[2].operator double();
+  attrs.op = op;
+  attrs.parsed = param;
+  SetAttrDict<op::NumpyWhereScalar2Param>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+MXNET_REGISTER_API("_npi.where")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  if (isScalar(args[1]) && isScalar(args[2])) {
+    _npi_where_scalar2(args, ret);
+  } else if (!isScalar(args[1]) && !isScalar(args[2])) {
+    _npi_where(args, ret);
+  } else {
+    _npi_where_scalar1(args, ret, isScalar(args[1]));
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_where_op-inl.h b/src/operator/numpy/np_where_op-inl.h
index 872ff18bfd02..a7011ff5c3fa 100644
--- a/src/operator/numpy/np_where_op-inl.h
+++ b/src/operator/numpy/np_where_op-inl.h
@@ -49,6 +49,11 @@ struct NumpyWhereScalarParam : public dmlc::Parameter<NumpyWhereScalarParam> {
     .set_default(0.0)
     .describe("The scalar value of x/y.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream scalar_s;
+    scalar_s << scalar;
+    (*dict)["scalar"] = scalar_s.str();
+  }
 };
 
 struct NumpyWhereScalar2Param : public dmlc::Parameter<NumpyWhereScalar2Param> {
@@ -61,6 +66,13 @@ struct NumpyWhereScalar2Param : public dmlc::Parameter<NumpyWhereScalar2Param> {
     .set_default(0.0)
     .describe("The scalar value of y.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream x_s, y_s;
+    x_s << x;
+    y_s << y;
+    (*dict)["x"] = x_s.str();
+    (*dict)["y"] = y_s.str();
+  }
 };
 
 template<int ndim>

From b816d4369cb8e7e32296aa9f5769480d68a07b4a Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Thu, 19 Mar 2020 02:10:41 +0800
Subject: [PATCH 12/44] ffi for roll/rot90 (#17861)

---
 benchmark/python/ffi/benchmark_ffi.py  |  2 +
 python/mxnet/_numpy_op_doc.py          | 64 -----------------------
 python/mxnet/ndarray/numpy/_op.py      | 70 +++++++++++++++++++++++++-
 python/mxnet/numpy/multiarray.py       | 68 ++++++++++++++++++++++++-
 python/mxnet/symbol/numpy/_symbol.py   | 37 +++++++++++++-
 src/api/operator/numpy/np_matrix_op.cc | 56 +++++++++++++++++++++
 src/operator/numpy/np_matrix_op-inl.h  | 18 +++++++
 src/operator/numpy/np_matrix_op.cc     |  4 +-
 src/operator/numpy/np_matrix_op.cu     |  2 +-
 9 files changed, 250 insertions(+), 71 deletions(-)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 2b9d8dfa58fe..25873588fc88 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -78,6 +78,8 @@ def prepare_workloads():
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
     OpArgMngr.add_workload("where", pool['2x3'], pool['2x3'], pool['2x1'])
     OpArgMngr.add_workload("may_share_memory", pool['2x3'][:0], pool['2x3'][:1])
+    OpArgMngr.add_workload("roll", pool["2x2"], 1, axis=0)
+    OpArgMngr.add_workload("rot90", pool["2x2"], 2)
 
 
 def benchmark_helper(f, *args, **kwargs):
diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 8dfc0867cdb2..3d80ce03b44e 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -538,70 +538,6 @@ def _np_reshape(a, newshape, order='C', out=None):
     """
 
 
-def _np_roll(a, shift, axis=None):
-    """
-    Roll array elements along a given axis.
-
-    Elements that roll beyond the last position are re-introduced at
-    the first.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input array.
-    shift : int or tuple of ints
-        The number of places by which elements are shifted.  If a tuple,
-        then `axis` must be a tuple of the same size, and each of the
-        given axes is shifted by the corresponding number.  If an int
-        while `axis` is a tuple of ints, then the same value is used for
-        all given axes.
-    axis : int or tuple of ints, optional
-        Axis or axes along which elements are shifted.  By default, the
-        array is flattened before shifting, after which the original
-        shape is restored.
-
-    Returns
-    -------
-    res : ndarray
-        Output array, with the same shape as `a`.
-
-    Notes
-    -----
-    Supports rolling over multiple dimensions simultaneously.
-
-    Examples
-    --------
-    >>> x = np.arange(10)
-    >>> np.roll(x, 2)
-    array([8., 9., 0., 1., 2., 3., 4., 5., 6., 7.])
-    >>> np.roll(x, -2)
-    array([2., 3., 4., 5., 6., 7., 8., 9., 0., 1.])
-
-    >>> x2 = np.reshape(x, (2,5))
-    >>> x2
-    array([[0., 1., 2., 3., 4.],
-           [5., 6., 7., 8., 9.]])
-    >>> np.roll(x2, 1)
-    array([[9., 0., 1., 2., 3.],
-           [4., 5., 6., 7., 8.]])
-    >>> np.roll(x2, -1)
-    array([[1., 2., 3., 4., 5.],
-           [6., 7., 8., 9., 0.]])
-    >>> np.roll(x2, 1, axis=0)
-    array([[5., 6., 7., 8., 9.],
-           [0., 1., 2., 3., 4.]])
-    >>> np.roll(x2, -1, axis=0)
-    array([[5., 6., 7., 8., 9.],
-           [0., 1., 2., 3., 4.]])
-    >>> np.roll(x2, 1, axis=1)
-    array([[4., 0., 1., 2., 3.],
-           [9., 5., 6., 7., 8.]])
-    >>> np.roll(x2, -1, axis=1)
-    array([[1., 2., 3., 4., 0.],
-           [6., 7., 8., 9., 5.]])
-   """
-
-
 def _np_trace(a, offset=0, axis1=0, axis2=1, out=None):
     """
     Return the sum along diagonals of the array.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index e235c54b8711..1ce1bcd1bdb1 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -43,7 +43,7 @@
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
            'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
-           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum',
+           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'where', 'bincount', 'pad', 'cumsum']
@@ -6307,6 +6307,72 @@ def less_equal(x1, x2, out=None):
                          _npi.greater_equal_scalar, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+def roll(a, shift, axis=None):
+    """
+    Roll array elements along a given axis.
+
+    Elements that roll beyond the last position are re-introduced at
+    the first.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    shift : int or tuple of ints
+        The number of places by which elements are shifted.  If a tuple,
+        then `axis` must be a tuple of the same size, and each of the
+        given axes is shifted by the corresponding number.  If an int
+        while `axis` is a tuple of ints, then the same value is used for
+        all given axes.
+    axis : int or tuple of ints, optional
+        Axis or axes along which elements are shifted.  By default, the
+        array is flattened before shifting, after which the original
+        shape is restored.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, with the same shape as `a`.
+
+    Notes
+    -----
+    Supports rolling over multiple dimensions simultaneously.
+
+    Examples
+    --------
+    >>> x = np.arange(10)
+    >>> np.roll(x, 2)
+    array([8., 9., 0., 1., 2., 3., 4., 5., 6., 7.])
+    >>> np.roll(x, -2)
+    array([2., 3., 4., 5., 6., 7., 8., 9., 0., 1.])
+
+    >>> x2 = np.reshape(x, (2,5))
+    >>> x2
+    array([[0., 1., 2., 3., 4.],
+           [5., 6., 7., 8., 9.]])
+    >>> np.roll(x2, 1)
+    array([[9., 0., 1., 2., 3.],
+           [4., 5., 6., 7., 8.]])
+    >>> np.roll(x2, -1)
+    array([[1., 2., 3., 4., 5.],
+           [6., 7., 8., 9., 0.]])
+    >>> np.roll(x2, 1, axis=0)
+    array([[5., 6., 7., 8., 9.],
+           [0., 1., 2., 3., 4.]])
+    >>> np.roll(x2, -1, axis=0)
+    array([[5., 6., 7., 8., 9.],
+           [0., 1., 2., 3., 4.]])
+    >>> np.roll(x2, 1, axis=1)
+    array([[4., 0., 1., 2., 3.],
+           [9., 5., 6., 7., 8.]])
+    >>> np.roll(x2, -1, axis=1)
+    array([[1., 2., 3., 4., 0.],
+           [6., 7., 8., 9., 5.]])
+   """
+    return _api_internal.roll(a, shift, axis)
+
+
 @set_module('mxnet.ndarray.numpy')
 def rot90(m, k=1, axes=(0, 1)):
     """
@@ -6350,7 +6416,7 @@ def rot90(m, k=1, axes=(0, 1)):
            [[5., 7.],
             [4., 6.]]])
     """
-    return _npi.rot90(m, k=k, axes=axes)
+    return _api_internal.rot90(m, k, axes)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index a5024253c162..0211f89e127c 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -66,7 +66,7 @@
            'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',
            'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',
            'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal',
-           'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum', 'true_divide', 'nonzero',
+           'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
            'pad', 'cumsum']
@@ -8130,6 +8130,72 @@ def less_equal(x1, x2, out=None):
     return _mx_nd_np.less_equal(x1, x2, out)
 
 
+@set_module('mxnet.numpy')
+def roll(a, shift, axis=None):
+    """
+    Roll array elements along a given axis.
+
+    Elements that roll beyond the last position are re-introduced at
+    the first.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    shift : int or tuple of ints
+        The number of places by which elements are shifted.  If a tuple,
+        then `axis` must be a tuple of the same size, and each of the
+        given axes is shifted by the corresponding number.  If an int
+        while `axis` is a tuple of ints, then the same value is used for
+        all given axes.
+    axis : int or tuple of ints, optional
+        Axis or axes along which elements are shifted.  By default, the
+        array is flattened before shifting, after which the original
+        shape is restored.
+
+    Returns
+    -------
+    res : ndarray
+        Output array, with the same shape as `a`.
+
+    Notes
+    -----
+    Supports rolling over multiple dimensions simultaneously.
+
+    Examples
+    --------
+    >>> x = np.arange(10)
+    >>> np.roll(x, 2)
+    array([8., 9., 0., 1., 2., 3., 4., 5., 6., 7.])
+    >>> np.roll(x, -2)
+    array([2., 3., 4., 5., 6., 7., 8., 9., 0., 1.])
+
+    >>> x2 = np.reshape(x, (2,5))
+    >>> x2
+    array([[0., 1., 2., 3., 4.],
+           [5., 6., 7., 8., 9.]])
+    >>> np.roll(x2, 1)
+    array([[9., 0., 1., 2., 3.],
+           [4., 5., 6., 7., 8.]])
+    >>> np.roll(x2, -1)
+    array([[1., 2., 3., 4., 5.],
+           [6., 7., 8., 9., 0.]])
+    >>> np.roll(x2, 1, axis=0)
+    array([[5., 6., 7., 8., 9.],
+           [0., 1., 2., 3., 4.]])
+    >>> np.roll(x2, -1, axis=0)
+    array([[5., 6., 7., 8., 9.],
+           [0., 1., 2., 3., 4.]])
+    >>> np.roll(x2, 1, axis=1)
+    array([[4., 0., 1., 2., 3.],
+           [9., 5., 6., 7., 8.]])
+    >>> np.roll(x2, -1, axis=1)
+    array([[1., 2., 3., 4., 0.],
+           [6., 7., 8., 9., 5.]])
+   """
+    return _mx_nd_np.roll(a, shift, axis=axis)
+
+
 @set_module('mxnet.numpy')
 def rot90(m, k=1, axes=(0, 1)):
     """
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index cdfa447c0ebd..e9868861d83d 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -48,7 +48,7 @@
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
            'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
-           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'rot90', 'einsum',
+           'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'where', 'bincount', 'pad', 'cumsum']
@@ -5802,6 +5802,41 @@ def less_equal(x1, x2, out=None):
                          _npi.greater_equal_scalar, out)
 
 
+@set_module('mxnet.symbol.numpy')
+def roll(a, shift, axis=None):
+    """
+    Roll array elements along a given axis.
+
+    Elements that roll beyond the last position are re-introduced at
+    the first.
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input array.
+    shift : int or tuple of ints
+        The number of places by which elements are shifted.  If a tuple,
+        then `axis` must be a tuple of the same size, and each of the
+        given axes is shifted by the corresponding number.  If an int
+        while `axis` is a tuple of ints, then the same value is used for
+        all given axes.
+    axis : int or tuple of ints, optional
+        Axis or axes along which elements are shifted.  By default, the
+        array is flattened before shifting, after which the original
+        shape is restored.
+
+    Returns
+    -------
+    res : _Symbol
+        Output array, with the same shape as `a`.
+
+    Notes
+    -----
+    Supports rolling over multiple dimensions simultaneously.
+    """
+    return _npi.roll(a, shift, axis=axis)
+
+
 @set_module('mxnet.symbol.numpy')
 def rot90(m, k=1, axes=(0, 1)):
     """
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index b4bb583c0511..36d06c7fc4cc 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -22,8 +22,10 @@
  * \brief Implementation of the API of functions in src/operator/tensor/matrix_op.cc
  */
 #include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
 #include "../../../operator/tensor/matrix_op-inl.h"
+#include "../../../operator/numpy/np_matrix_op-inl.h"
 
 namespace mxnet {
 
@@ -86,4 +88,58 @@ MXNET_REGISTER_API("_npi.split")
   *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
 });
 
+MXNET_REGISTER_API("_npi.roll")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_roll");
+  nnvm::NodeAttrs attrs;
+  op::NumpyRollParam param;
+  if (args[1].type_code() == kNull) {
+    param.shift = dmlc::nullopt;
+  } else if (args[1].type_code() == kDLInt) {
+    param.shift = TShape(1, args[1].operator int64_t());
+  } else {
+    param.shift = TShape(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else if (args[2].type_code() == kDLInt) {
+    param.axis = TShape(1, args[2].operator int64_t());
+  } else {
+    param.axis = TShape(args[2].operator ObjectRef());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyRollParam>(&attrs);
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.rot90")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_rot90");
+  nnvm::NodeAttrs attrs;
+  op::NumpyRot90Param param;
+  param.k = args[1].operator int();
+  if (args[2].type_code() == kNull) {
+    param.axes = dmlc::nullopt;
+  } else if (args[2].type_code() == kDLInt) {
+    param.axes = TShape(1, args[2].operator int64_t());
+  } else {
+    param.axes = TShape(args[2].operator ObjectRef());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyRot90Param>(&attrs);
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index 593a698dc93a..0bbe263cfc76 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -302,6 +302,13 @@ struct NumpyRollParam : public dmlc::Parameter<NumpyRollParam> {
     .describe("Axis or axes along which elements are shifted. By default, the array is flattened"
               "before shifting, after which the original shape is restored.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream shift_s, axis_s;
+    shift_s << shift;
+    axis_s << axis;
+    (*dict)["shift"] = shift_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 template<int req>
@@ -605,6 +612,13 @@ struct NumpyRot90Param : public dmlc::Parameter<NumpyRot90Param> {
     .set_default(dmlc::optional<mxnet::TShape>())
     .describe(" The array is rotated in the plane defined by the axes. Axes must be different.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream k_s, axes_s;
+    k_s << k;
+    axes_s << axes;
+    (*dict)["k"] = k_s.str();
+    (*dict)["axes"] = axes_s.str();
+  }
 };
 
 struct rot90reverse {
@@ -852,6 +866,10 @@ inline void HSplitOpForward(const nnvm::NodeAttrs &attrs,
   } else {
     real_axis = 0;
   }
+  if (param.sections > 0) {
+    CHECK_EQ(inputs[0].shape_[real_axis] % param.sections, 0U)
+      << "ValueError: array split does not result in an equal division";
+  }
   SplitOpForwardImpl<xpu>(attrs, ctx, inputs, req, outputs, real_axis);
 }
 
diff --git a/src/operator/numpy/np_matrix_op.cc b/src/operator/numpy/np_matrix_op.cc
index b6b8ff332db8..e9d269dd54d6 100644
--- a/src/operator/numpy/np_matrix_op.cc
+++ b/src/operator/numpy/np_matrix_op.cc
@@ -1155,7 +1155,7 @@ inline bool NumpyRollShape(const nnvm::NodeAttrs& attrs,
   return ElemwiseShape<1, 1>(attrs, in_attrs, out_attrs);
 }
 
-NNVM_REGISTER_OP(_np_roll)
+NNVM_REGISTER_OP(_npi_roll)
 .set_num_inputs(1)
 .set_num_outputs(1)
 .set_attr_parser(ParamParser<NumpyRollParam>)
@@ -1180,7 +1180,7 @@ NNVM_REGISTER_OP(_np_roll)
      os1 << dmlc::optional<mxnet::TShape>(shifts);
      std::ostringstream os2;
      os2 << param.axis;
-     return MakeNonlossGradNode("_np_roll", n, ograds, {},
+     return MakeNonlossGradNode("_npi_roll", n, ograds, {},
                                 {{"shift", os1.str()}, {"axis", os2.str()}});
 })
 .set_attr<FResourceRequest>("FResourceRequest",
diff --git a/src/operator/numpy/np_matrix_op.cu b/src/operator/numpy/np_matrix_op.cu
index 335f4fd984d6..c9e896bc5b57 100644
--- a/src/operator/numpy/np_matrix_op.cu
+++ b/src/operator/numpy/np_matrix_op.cu
@@ -71,7 +71,7 @@ NNVM_REGISTER_OP(_npi_column_stack)
 NNVM_REGISTER_OP(_backward_np_column_stack)
 .set_attr<FCompute>("FCompute<gpu>", NumpyColumnStackBackward<gpu>);
 
-NNVM_REGISTER_OP(_np_roll)
+NNVM_REGISTER_OP(_npi_roll)
 .set_attr<FCompute>("FCompute<gpu>", NumpyRollCompute<gpu>);
 
 template<>

From 34b4708aad55cd1266dc489a69815d5c7ca267fb Mon Sep 17 00:00:00 2001
From: JiangZhaoh <54654391+JiangZhaoh@users.noreply.github.com>
Date: Thu, 9 Apr 2020 07:07:01 +0800
Subject: [PATCH 13/44] [Numpy] allow mix integer dtypes for power/add/multiply
 (#17921)

* resolution

* fix sanity error

* remove func 'is_integer'
---
 .../numpy/np_elemwise_broadcast_op.cc         |  4 ++
 src/operator/numpy/np_elemwise_broadcast_op.h | 68 ++++++++++++++++++-
 tests/python/unittest/test_numpy_op.py        |  9 +++
 3 files changed, 79 insertions(+), 2 deletions(-)

diff --git a/src/operator/numpy/np_elemwise_broadcast_op.cc b/src/operator/numpy/np_elemwise_broadcast_op.cc
index ae285caa9094..0ee677adbc13 100644
--- a/src/operator/numpy/np_elemwise_broadcast_op.cc
+++ b/src/operator/numpy/np_elemwise_broadcast_op.cc
@@ -76,6 +76,10 @@ bool NumpyBinaryMixedPrecisionType(const nnvm::NodeAttrs& attrs,
     [](const NodeAttrs& attrs){                                                \
       return std::vector<std::pair<int, int> >{{0, 0}, {1, 0}};                \
     })                                                                         \
+  .set_attr<FResourceRequest>("FResourceRequest",                              \
+    [](const NodeAttrs& attrs) {                                               \
+      return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};        \
+    })                                                                         \
   .add_argument("lhs", "NDArray-or-Symbol", "First input to the function")     \
   .add_argument("rhs", "NDArray-or-Symbol", "Second input to the function")
 #else
diff --git a/src/operator/numpy/np_elemwise_broadcast_op.h b/src/operator/numpy/np_elemwise_broadcast_op.h
index d58245a798e5..a0e204318839 100644
--- a/src/operator/numpy/np_elemwise_broadcast_op.h
+++ b/src/operator/numpy/np_elemwise_broadcast_op.h
@@ -153,7 +153,6 @@ void MixedBinaryElemwiseCompute(const nnvm::NodeAttrs& attrs,
   const TBlob& lhs = inputs[0];
   const TBlob& rhs = inputs[1];
   const TBlob& out = outputs[0];
-
   if (common::is_float(lhs.type_flag_) && common::is_float(rhs.type_flag_)) {
     if (lhs.type_flag_ == out.type_flag_) {
       MixedAllRealBinaryElemwiseCompute<xpu, ROP>(attrs.op->name, ctx, lhs, rhs, out, req[0]);
@@ -252,7 +251,6 @@ void MixedBinaryBroadcastCompute(const nnvm::NodeAttrs& attrs,
   mxnet::TShape new_lshape, new_rshape, new_oshape;
   int ndim = BinaryBroadcastShapeCompact(lhs.shape_, rhs.shape_, out.shape_,
                                          &new_lshape, &new_rshape, &new_oshape);
-
   if (!ndim) {
     MixedBinaryElemwiseCompute<xpu, LOP, ROP>(attrs, ctx, inputs, req, outputs);
   } else {
@@ -290,6 +288,27 @@ void MixedBinaryBroadcastCompute(const nnvm::NodeAttrs& attrs,
           });
         }
       });
+    } else if (!common::is_float(lhs.type_flag_) && !common::is_float(rhs.type_flag_)) {
+      TBlob temp_tblob;
+      if (lhs.type_flag_ == out.type_flag_) {
+        MXNET_INT_TYPE_SWITCH(lhs.type_flag_, LType, {
+          Tensor<xpu, 1, LType> temp_tensor =
+            ctx.requested[0].get_space_typed<xpu, 1, LType>(Shape1(rhs.Size()), s);
+          temp_tblob = TBlob(temp_tensor);
+        });
+        CastCompute<xpu>(attrs, ctx, {rhs}, {kWriteTo}, {temp_tblob});
+        BinaryBroadcastCompute<xpu, OP>(
+          attrs, ctx, {lhs, temp_tblob.reshape(rhs.shape_)}, req, outputs);
+      } else {
+        MXNET_INT_TYPE_SWITCH(rhs.type_flag_, RType, {
+          Tensor<xpu, 1, RType> temp_tensor =
+            ctx.requested[0].get_space_typed<xpu, 1, RType>(Shape1(lhs.Size()), s);
+          temp_tblob = TBlob(temp_tensor);
+        });
+        CastCompute<xpu>(attrs, ctx, {lhs}, {kWriteTo}, {temp_tblob});
+        BinaryBroadcastCompute<xpu, OP>(
+          attrs, ctx, {temp_tblob.reshape(lhs.shape_), rhs}, req, outputs);
+      }
     } else {
       PrintErrorMessage(attrs.op->name, lhs.type_flag_, rhs.type_flag_);
     }
@@ -320,6 +339,27 @@ void MixedBinaryBroadcastCompute(const nnvm::NodeAttrs& attrs,
       BinaryBroadcastCompute<xpu, OP>(
         attrs, ctx, {temp_tblob.reshape(lhs.shape_), rhs}, req, outputs);
     }
+  } else if (!common::is_float(lhs.type_flag_) && !common::is_float(rhs.type_flag_)) {
+    TBlob temp_tblob;
+    if (lhs.type_flag_ == out.type_flag_) {
+      MXNET_INT_TYPE_SWITCH(lhs.type_flag_, LType, {
+        Tensor<xpu, 1, LType> temp_tensor =
+          ctx.requested[0].get_space_typed<xpu, 1, LType>(Shape1(rhs.Size()), s);
+        temp_tblob = TBlob(temp_tensor);
+      });
+      CastCompute<xpu>(attrs, ctx, {rhs}, {kWriteTo}, {temp_tblob});
+      BinaryBroadcastCompute<xpu, OP>(
+        attrs, ctx, {lhs, temp_tblob.reshape(rhs.shape_)}, req, outputs);
+    } else {
+      MXNET_INT_TYPE_SWITCH(rhs.type_flag_, RType, {
+        Tensor<xpu, 1, RType> temp_tensor =
+          ctx.requested[0].get_space_typed<xpu, 1, RType>(Shape1(lhs.Size()), s);
+        temp_tblob = TBlob(temp_tensor);
+      });
+      CastCompute<xpu>(attrs, ctx, {lhs}, {kWriteTo}, {temp_tblob});
+      BinaryBroadcastCompute<xpu, OP>(
+        attrs, ctx, {temp_tblob.reshape(lhs.shape_), rhs}, req, outputs);
+    }
   } else {
     PrintErrorMessage(attrs.op->name, lhs.type_flag_, rhs.type_flag_);
   }
@@ -384,6 +424,30 @@ void NumpyBinaryBroadcastComputeWithBool(const nnvm::NodeAttrs& attrs,
     BinaryBroadcastComputeWithBool<xpu, OP>(attrs, ctx, inputs, req, outputs);
     return;
   }
+  if (!common::is_float(lhs.type_flag_) && !common::is_float(rhs.type_flag_)) {
+    Stream<xpu> *s = ctx.get_stream<xpu>();
+    TBlob temp_tblob;
+    if (lhs.type_flag_ == out.type_flag_) {
+      MXNET_INT_TYPE_SWITCH(lhs.type_flag_, LType, {
+        Tensor<xpu, 1, LType> temp_tensor =
+          ctx.requested[0].get_space_typed<xpu, 1, LType>(Shape1(rhs.Size()), s);
+        temp_tblob = TBlob(temp_tensor);
+      });
+      CastCompute<xpu>(attrs, ctx, {rhs}, {kWriteTo}, {temp_tblob});
+      BinaryBroadcastCompute<xpu, OP>(
+        attrs, ctx, {lhs, temp_tblob.reshape(rhs.shape_)}, req, outputs);
+    } else {
+      MXNET_INT_TYPE_SWITCH(rhs.type_flag_, RType, {
+        Tensor<xpu, 1, RType> temp_tensor =
+          ctx.requested[0].get_space_typed<xpu, 1, RType>(Shape1(lhs.Size()), s);
+        temp_tblob = TBlob(temp_tensor);
+      });
+      CastCompute<xpu>(attrs, ctx, {lhs}, {kWriteTo}, {temp_tblob});
+      BinaryBroadcastCompute<xpu, OP>(
+        attrs, ctx, {temp_tblob.reshape(lhs.shape_), rhs}, req, outputs);
+    }
+    return;
+  }
 
 #ifndef _WIN32
   MixedBinaryBroadcastCompute<xpu, OP, LOP, ROP>(attrs, ctx, inputs, req, outputs);
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 92cd06f3317e..cdba6f41b0df 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -2490,6 +2490,8 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
                                 use_broadcast=False, equal_nan=True)
 
             if lgrad:
+                if (ltype in itypes) and (rtype in itypes):
+                    continue
                 y.backward()
                 if ltype not in itypes:
                     assert_almost_equal(mx_test_x1.grad.asnumpy(),
@@ -2544,6 +2546,13 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
                     continue
                 check_mixed_precision_binary_func(func, low, high, lshape, rshape, lgrad, rgrad, type1, type2)
 
+            if func == 'subtract':
+                continue
+            for type1, type2 in itertools.product(itypes, itypes):
+                if type1 == type2:
+                    continue
+                check_mixed_precision_binary_func(func, low, high, lshape, rshape, lgrad, rgrad, type1, type2)
+
 
 @with_seed()
 @use_np

From aedf66b8cb9f3620524a16bee029d65ff5a24948 Mon Sep 17 00:00:00 2001
From: Xingjian Shi <xshiab@connect.ust.hk>
Date: Sat, 23 May 2020 18:43:34 -0700
Subject: [PATCH 14/44] fix true_divide (#18393)

---
 src/operator/numpy/np_true_divide-inl.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/operator/numpy/np_true_divide-inl.h b/src/operator/numpy/np_true_divide-inl.h
index 0bc60a08803e..be2ce51506a1 100644
--- a/src/operator/numpy/np_true_divide-inl.h
+++ b/src/operator/numpy/np_true_divide-inl.h
@@ -121,7 +121,7 @@ void TrueDivideElemwiseCompute(const nnvm::NodeAttrs &attrs,
     // Case when types of the 2 input tensors are different
     if (common::is_float(lhs.type_flag_) && common::is_float(rhs.type_flag_)) {
       // both lhs and rhs are float types, output type is the more precise one
-      LOG(ERROR) << "not implemented yet...";
+      LOG(FATAL) << "not implemented yet...";
     } else if (common::is_float(lhs.type_flag_) || common::is_float(rhs.type_flag_)) {
       // one is float type, the other is integer type, the output type should be the same as float
       CHECK_EQ(out.type_flag_,
@@ -150,14 +150,14 @@ void TrueDivideElemwiseCompute(const nnvm::NodeAttrs &attrs,
       }
     } else {
       // lhs is integer type, rhs is integer type, output type should be float
-      LOG(ERROR) << "not implemented yet...";
+      LOG(FATAL) << "not implemented yet...";
     }
 #else
     // Windows case: using temp space for casting the type
     // Case when types of the 2 input tensors are different
     if (common::is_float(lhs.type_flag_) && common::is_float(rhs.type_flag_)) {
       // both lhs and rhs are float types, output type is the more precise one
-      LOG(ERROR) << "not implemented yet...";
+      LOG(FATAL) << "not implemented yet...";
     } else if (common::is_float(lhs.type_flag_) || common::is_float(rhs.type_flag_)) {
       // lhs is float type, rhs is integer type, the output type should be the same as lhs
       CHECK_EQ(out.type_flag_,
@@ -187,7 +187,7 @@ void TrueDivideElemwiseCompute(const nnvm::NodeAttrs &attrs,
       }
     } else {
       // lhs is integer type, rhs is integer type, output type should be float
-      LOG(ERROR) << "not implemented yet...";
+      LOG(FATAL) << "not implemented yet...";
     }
 #endif
   }
@@ -241,7 +241,7 @@ void TrueDivideBroadcastCompute(const nnvm::NodeAttrs& attrs,
       } else {
         if (common::is_float(lhs.type_flag_) && common::is_float(rhs.type_flag_)) {
           // lhs and rhs have different float types, the output is the more precise one
-          LOG(ERROR) << "not implemented yet...";
+          LOG(FATAL) << "not implemented yet...";
         } else if (common::is_float(lhs.type_flag_) || common::is_float(rhs.type_flag_)) {
           // one of lhs and rhs is float, the output is the same type as the float one
           if (common::is_float(lhs.type_flag_)) {
@@ -269,7 +269,7 @@ void TrueDivideBroadcastCompute(const nnvm::NodeAttrs& attrs,
           }
         } else {
           // lhs and rhs have different integer types, the output is float type
-          LOG(ERROR) << "not implemented yet...";
+          LOG(FATAL) << "not implemented yet...";
         }
       }
     });
@@ -302,7 +302,7 @@ void TrueDivideBroadcastCompute(const nnvm::NodeAttrs& attrs,
     } else {
       if (common::is_float(lhs.type_flag_) && common::is_float(rhs.type_flag_)) {
         // lhs and rhs have different float types, the output is the more precise one
-        LOG(ERROR) << "not implemented yet...";
+        LOG(FATAL) << "not implemented yet...";
       } else if (common::is_float(lhs.type_flag_) || common::is_float(rhs.type_flag_)) {
         // one of lhs and rhs is float, the output is the same type as the float one
         TBlob temp_tblob;
@@ -333,7 +333,7 @@ void TrueDivideBroadcastCompute(const nnvm::NodeAttrs& attrs,
         }
       } else {
         // lhs and rhs have different integer types, the output is float type
-        LOG(ERROR) << "not implemented yet...";
+        LOG(FATAL) << "not implemented yet...";
       }
     }
 #endif

From f862579c3bc94a83a22901db5aeb94f8d6d9152c Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Thu, 19 Mar 2020 02:37:29 +0800
Subject: [PATCH 15/44] * FFI for np.argmax and np.argmin (#17843)

* impl - FFI for np_indices

* fix - use MXNetTypeWithBool2String

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py         |  3 +
 python/mxnet/ndarray/numpy/_op.py             | 10 +-
 .../numpy/np_broadcast_reduce_op_index.cc     | 98 +++++++++++++++++++
 src/api/operator/numpy/np_init_op.cc          | 32 ++++++
 src/operator/numpy/np_init_op.h               |  8 ++
 src/operator/tensor/broadcast_reduce_op.h     |  7 ++
 6 files changed, 154 insertions(+), 4 deletions(-)
 create mode 100644 src/api/operator/numpy/np_broadcast_reduce_op_index.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 25873588fc88..1983de594b28 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -59,6 +59,9 @@ def prepare_workloads():
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
+    OpArgMngr.add_workload("argmax", pool['3x2'], axis=-1)
+    OpArgMngr.add_workload("argmin", pool['3x2'], axis=-1)
+    OpArgMngr.add_workload("indices", dimensions=(1, 2, 3))
     OpArgMngr.add_workload("subtract", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("multiply", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("mod", pool['2x2'], pool['2x2'])
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 1ce1bcd1bdb1..32519d142c1e 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -4529,7 +4529,7 @@ def argmax(a, axis=None, out=None):
     >>> b
     array([2., 2.])
     """
-    return _npi.argmax(a, axis=axis, keepdims=False, out=out)
+    return _api_internal.argmax(a, axis, False, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4597,7 +4597,7 @@ def argmin(a, axis=None, out=None):
     >>> b
     array([0., 0.])
     """
-    return _npi.argmin(a, axis=axis, keepdims=False, out=out)
+    return _api_internal.argmin(a, axis, False, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4945,8 +4945,10 @@ def indices(dimensions, dtype=_np.int32, ctx=None):
     """
     if isinstance(dimensions, (tuple, list)):
         if ctx is None:
-            ctx = current_context()
-        return _npi.indices(dimensions=dimensions, dtype=dtype, ctx=ctx)
+            ctx = str(current_context())
+        else:
+            ctx = str(ctx)
+        return _api_internal.indices(dimensions, dtype, ctx)
     else:
         raise ValueError("The dimensions must be sequence of ints")
 # pylint: enable=redefined-outer-name
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_index.cc b/src/api/operator/numpy/np_broadcast_reduce_op_index.cc
new file mode 100644
index 000000000000..aa24246f693d
--- /dev/null
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_index.cc
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_broadcast_reduce_op_index.cc
+ * \brief Implementation of the API of functions in
+          src/operator/numpy/np_broadcast_reduce_op_index.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/tensor/broadcast_reduce_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.argmax")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_argmax");
+  nnvm::NodeAttrs attrs;
+  op::ReduceAxisParam param;
+  // param.axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[1].operator int();
+  }
+  // param.keepdims
+  param.keepdims = args[2].operator bool();
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::ReduceAxisParam>(&attrs);
+  // inputs
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  // outputs
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+MXNET_REGISTER_API("_npi.argmin")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_argmin");
+  nnvm::NodeAttrs attrs;
+  op::ReduceAxisParam param;
+  // param.axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[1].operator int();
+  }
+  // param.keepdims
+  param.keepdims = args[2].operator bool();
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::ReduceAxisParam>(&attrs);
+  // inputs
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  // outputs
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_init_op.cc b/src/api/operator/numpy/np_init_op.cc
index 4f7c6e497616..c339fb505029 100644
--- a/src/api/operator/numpy/np_init_op.cc
+++ b/src/api/operator/numpy/np_init_op.cc
@@ -26,6 +26,7 @@
 #include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
 #include "../../../operator/tensor/init_op.h"
+#include "../../../operator/numpy/np_init_op.h"
 
 namespace mxnet {
 
@@ -88,4 +89,35 @@ MXNET_REGISTER_API("_npi.full_like")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.indices")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_indices");
+  nnvm::NodeAttrs attrs;
+  op::IndicesOpParam param;
+  // param.dimensions
+  if (args[0].type_code() == kDLInt) {
+    param.dimensions = TShape(1, args[0].operator int64_t());
+  } else {
+    param.dimensions = TShape(args[0].operator ObjectRef());
+  }
+  // param.dtype
+  if (args[1].type_code() == kNull) {
+    param.dtype = mshadow::kInt32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::IndicesOpParam>(&attrs);
+  // param.ctx
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  int num_inputs = 0;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_init_op.h b/src/operator/numpy/np_init_op.h
index cfc2941ecd28..e92af5fc4544 100644
--- a/src/operator/numpy/np_init_op.h
+++ b/src/operator/numpy/np_init_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include "../tensor/init_op.h"
 #include "../tensor/elemwise_unary_op.h"
+#include "../../api/operator/op_utils.h"
 
 
 namespace mxnet {
@@ -79,6 +80,13 @@ struct IndicesOpParam : public dmlc::Parameter<IndicesOpParam> {
     .describe("Context of output, in format [cpu|gpu|cpu_pinned](n)."
               "Only used for imperative calls.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream dimensions_s, dtype_s;
+    dimensions_s << dimensions;
+    dtype_s << dtype;
+    (*dict)["dimensions"] = dimensions_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 inline bool NumpyRangeShape(const nnvm::NodeAttrs& attrs,
diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h
index b06442932a78..03aa8b932dff 100644
--- a/src/operator/tensor/broadcast_reduce_op.h
+++ b/src/operator/tensor/broadcast_reduce_op.h
@@ -108,6 +108,13 @@ struct ReduceAxisParam : public dmlc::Parameter<ReduceAxisParam> {
       .describe("If this is set to `True`, the reduced axis is left "
                 "in the result as dimension with size one.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, keepdims_s;
+    axis_s << axis;
+    keepdims_s << keepdims;
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["keepdims"] = keepdims_s.str();
+  }
 };
 
 enum PickOpMode {kWrap, kClip};

From f00c3b5a31129d6565c6e52c404712d72544116b Mon Sep 17 00:00:00 2001
From: Hao Jin <hjjn.amzn@gmail.com>
Date: Thu, 5 Mar 2020 20:34:56 -0800
Subject: [PATCH 16/44] add alias for np.__version__, np._NoValue and np.dtype
 (#17777)

---
 python/mxnet/numpy/fallback.py                |  7 ++++
 python/mxnet/numpy/multiarray.py              | 36 +++++++++++++++++--
 .../unittest/test_numpy_interoperability.py   |  3 +-
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/python/mxnet/numpy/fallback.py b/python/mxnet/numpy/fallback.py
index b98d377e4cd2..54df715b60c0 100644
--- a/python/mxnet/numpy/fallback.py
+++ b/python/mxnet/numpy/fallback.py
@@ -20,7 +20,10 @@
 
 import numpy as onp
 
+
 __all__ = [
+    '__version__',
+    '_NoValue',
     'allclose',
     'alltrue',
     'apply_along_axis',
@@ -37,6 +40,7 @@
     'cov',
     'digitize',
     'divmod',
+    'dtype',
     'extract',
     'float_power',
     'frexp',
@@ -106,6 +110,8 @@
     'vander',
 ]
 
+__version__ = onp.__version__
+_NoValue = onp._NoValue
 allclose = onp.allclose
 alltrue = onp.alltrue
 apply_along_axis = onp.apply_along_axis
@@ -122,6 +128,7 @@
 cov = onp.cov
 digitize = onp.digitize
 divmod = onp.divmod
+dtype = onp.dtype
 extract = onp.extract
 float_power = onp.float_power
 frexp = onp.frexp
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 0211f89e127c..72e367aa7894 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -2230,6 +2230,7 @@ def empty(shape, dtype=_np.float32, order='C', ctx=None):  # pylint: disable=red
     return ndarray(handle=_new_alloc_handle(shape, ctx, False, dtype))
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def array(object, dtype=None, ctx=None):
     """
@@ -2286,6 +2287,7 @@ def array(object, dtype=None, ctx=None):
     else:
         ret[:] = object
     return ret
+# pylint: enable=redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -2497,6 +2499,7 @@ def full(shape, fill_value, dtype=None, order='C', ctx=None, out=None):
 # pylint: enable=too-many-arguments, redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pylint: disable=W0621
     """
@@ -2554,8 +2557,10 @@ def empty_like(prototype, dtype=None, order='C', subok=False, shape=None): # pyl
            [2.0e-323, 2.5e-323, 3.0e-323]])
     """
     return _mx_nd_np.empty_like(prototype, dtype=dtype, order=order, subok=subok, shape=shape)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def all(a, axis=None, out=None, keepdims=False):
     """
@@ -2692,6 +2697,7 @@ def identity(n, dtype=None, ctx=None):
            [0., 0., 1.]])
     """
     return _mx_nd_np.identity(n, dtype, ctx)
+# pylint: enable=redefined-outer-name
 
 
 # pylint: disable=redefined-outer-name
@@ -5111,6 +5117,7 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, density=None):
     return _mx_nd_np.histogram(a, bins=bins, range=range, normed=normed, weights=weights, density=density)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
     """
@@ -5146,8 +5153,10 @@ def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
            [0., 0., 0.]])
     """
     return _mx_nd_np.eye(N, M, k, dtype, **kwargs)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0, ctx=None):  # pylint: disable=too-many-arguments
     r"""
@@ -5233,9 +5242,10 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis
       GPU.
     """
     return _mx_nd_np.linspace(start, stop, num, endpoint, retstep, dtype, axis, ctx)
+# pylint: enable=redefined-outer-name
 
 
-# pylint: disable=too-many-arguments
+# pylint: disable=too-many-arguments, redefined-outer-name
 @set_module('mxnet.numpy')
 def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0, ctx=None):
     r"""Return numbers spaced evenly on a log scale.
@@ -5310,7 +5320,7 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0,
     array([ 100.     ,  215.44347,  464.15887, 1000.     ], ctx=gpu(0))
     """
     return _mx_nd_np.logspace(start, stop, num, endpoint, base, dtype, axis, ctx=ctx)
-# pylint: enable=too-many-arguments
+# pylint: enable=too-many-arguments, redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -5470,6 +5480,7 @@ def tril(m, k=0):
     return _mx_nd_np.tril(m, k)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def arange(start, stop=None, step=1, dtype=None, ctx=None):
     """Return evenly spaced values within a given interval.
@@ -5521,6 +5532,7 @@ def arange(start, stop=None, step=1, dtype=None, ctx=None):
     array([3., 5.])
     """
     return _mx_nd_np.arange(start, stop, step, dtype, ctx)
+# pylint: enable=redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -6508,6 +6520,7 @@ def average(a, axis=None, weights=None, returned=False, out=None):
     return _mx_nd_np.average(a, axis=axis, weights=weights, returned=returned, out=out)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ
     """
@@ -6565,9 +6578,10 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable
     array(0.55)
     """
     return _npi.mean(a, axis=axis, dtype=dtype, keepdims=keepdims, out=out)
+# pylint: enable=redefined-outer-name
 
 
-
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments
     """
@@ -6633,6 +6647,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint:
     array(0.45, dtype=float64)
     """
     return _npi.std(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
+# pylint: enable=redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -6684,6 +6699,7 @@ def delete(arr, obj, axis=None):
     return _mx_nd_np.delete(arr, obj, axis=axis)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint: disable=too-many-arguments
     """
@@ -7022,6 +7038,7 @@ def diag_indices_from(arr):
     return _mx_nd_np.diag_indices_from(arr)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def hanning(M, dtype=_np.float32, ctx=None):
     r"""Return the Hanning window.
@@ -7100,8 +7117,10 @@ def hanning(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     return _mx_nd_np.hanning(M, dtype=dtype, ctx=ctx)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def hamming(M, dtype=_np.float32, ctx=None):
     r"""Return the hamming window.
@@ -7178,8 +7197,10 @@ def hamming(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     return _mx_nd_np.hamming(M, dtype=dtype, ctx=ctx)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def blackman(M, dtype=_np.float32, ctx=None):
     r"""Return the Blackman window.
@@ -7254,6 +7275,7 @@ def blackman(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     return _mx_nd_np.blackman(M, dtype=dtype, ctx=ctx)
+# pylint: enable=redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -9091,6 +9113,7 @@ def resize(a, new_shape):
     return _mx_nd_np.resize(a, new_shape)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def full_like(a, fill_value, dtype=None, order='C', ctx=None, out=None): # pylint: disable=too-many-arguments
     """
@@ -9143,8 +9166,10 @@ def full_like(a, fill_value, dtype=None, order='C', ctx=None, out=None): # pylin
     array([0.1, 0.1, 0.1, 0.1, 0.1, 0.1])
     """
     return _mx_nd_np.full_like(a, fill_value=fill_value, dtype=dtype, order=order, ctx=ctx, out=out)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def zeros_like(a, dtype=None, order='C', ctx=None, out=None):
     """
@@ -9199,8 +9224,10 @@ def zeros_like(a, dtype=None, order='C', ctx=None, out=None):
     array([0., 0., 0.], dtype=float64)
     """
     return _mx_nd_np.full_like(a, fill_value=0, dtype=dtype, order=order, ctx=ctx, out=ctx)
+# pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def ones_like(a, dtype=None, order='C', ctx=None, out=None):
     """
@@ -9255,6 +9282,7 @@ def ones_like(a, dtype=None, order='C', ctx=None, out=None):
     array([1., 1., 1.], dtype=float64)
     """
     return _mx_nd_np.full_like(a, fill_value=1, dtype=dtype, order=order, ctx=ctx, out=out)
+# pylint: enable=redefined-outer-name
 
 
 @set_module('mxnet.numpy')
@@ -9860,6 +9888,7 @@ def pad(x, pad_width=None, mode="constant", **kwargs): # pylint: disable=too-man
     return _mx_nd_np.pad(x, pad_width, mode, **kwargs)
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def cumsum(a, axis=None, dtype=None, out=None):
     """
@@ -9910,3 +9939,4 @@ def cumsum(a, axis=None, dtype=None, out=None):
            [ 4,  9, 15]])
     """
     return _mx_nd_np.cumsum(a, axis=axis, dtype=dtype, out=out)
+# pylint: enable=redefined-outer-name
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 298d565dc237..2d1710eeedc9 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -3031,7 +3031,8 @@ def check_interoperability(op_list):
     for name in op_list:
         if name in _TVM_OPS and not is_op_runnable():
             continue
-        if name in ['shares_memory', 'may_share_memory', 'empty_like']:  # skip list
+        if name in ['shares_memory', 'may_share_memory', 'empty_like',
+                    '__version__', 'dtype', '_NoValue']:  # skip list
             continue
         if name in ['full_like', 'zeros_like', 'ones_like'] and \
                 StrictVersion(platform.python_version()) < StrictVersion('3.0.0'):

From 27d5008996e97f4abc6cbf52382461f7800b27de Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Tue, 24 Mar 2020 15:26:32 +0800
Subject: [PATCH 17/44] [Numpy] FFI for diag/diagonal/diag_indices_from
 (#17789)

* ffi_diag/diagonal/diag_indices_from

* sanity && benchmark
---
 benchmark/python/ffi/benchmark_ffi.py  |  12 +++
 python/mxnet/ndarray/numpy/_op.py      | 106 +++++++++++++++++++++++--
 python/mxnet/numpy/multiarray.py       |  96 +++++++++++++++++++++-
 python/mxnet/symbol/numpy/_symbol.py   |  56 ++++++++++++-
 src/api/operator/numpy/np_matrix_op.cc |  49 ++++++++++++
 src/operator/numpy/np_matrix_op-inl.h  |  44 ++++++----
 src/operator/numpy/np_matrix_op.cc     |  12 +--
 src/operator/numpy/np_matrix_op.cu     |   8 +-
 8 files changed, 349 insertions(+), 34 deletions(-)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 1983de594b28..ac7793706a22 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -81,6 +81,18 @@ def prepare_workloads():
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
     OpArgMngr.add_workload("where", pool['2x3'], pool['2x3'], pool['2x1'])
     OpArgMngr.add_workload("may_share_memory", pool['2x3'][:0], pool['2x3'][:1])
+    OpArgMngr.add_workload("diag", pool['2x2'], k=1)
+    OpArgMngr.add_workload("diagonal", pool['2x2x2'], offset=-1, axis1=0, axis2=1)
+    OpArgMngr.add_workload("diag_indices_from", pool['2x2'])
+    OpArgMngr.add_workload("bincount", dnp.arange(3, dtype=int), pool['3'], minlength=4)
+    OpArgMngr.add_workload("percentile", pool['2x2x2'], 80, axis=0, out=pool['2x2'],\
+                           interpolation='midpoint')
+    OpArgMngr.add_workload("quantile", pool['2x2x2'], 0.8, axis=0, out=pool['2x2'],\
+                           interpolation='midpoint')
+    OpArgMngr.add_workload("all", pool['2x2x2'], axis=(0, 1),\
+                           out=dnp.array([False, False], dtype=bool), keepdims=False)
+    OpArgMngr.add_workload("any", pool['2x2x2'], axis=(0, 1),\
+                           out=dnp.array([False, False], dtype=bool), keepdims=False)
     OpArgMngr.add_workload("roll", pool["2x2"], 1, axis=0)
     OpArgMngr.add_workload("rot90", pool["2x2"], 2)
 
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 32519d142c1e..fdfefdd6793a 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -46,7 +46,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
-           'where', 'bincount', 'pad', 'cumsum']
+           'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5058,6 +5058,7 @@ def ravel(x, order='C'):
         raise TypeError('type {} not supported'.format(str(type(x))))
 
 
+@set_module('mxnet.ndarray.numpy')
 def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-name
     """
     Converts a flat index or array of flat indices into a tuple of coordinate arrays.
@@ -5087,11 +5088,7 @@ def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-
     if order == 'C':
         if isinstance(indices, numeric_types):
             return _np.unravel_index(indices, shape)
-        ret = _npi.unravel_index_fallback(indices, shape=shape)
-        ret_list = []
-        for item in ret:
-            ret_list += [item]
-        return tuple(ret_list)
+        return tuple(_npi.unravel_index_fallback(indices, shape=shape))
     else:
         raise NotImplementedError('Do not support column-major (Fortran-style) order at this moment')
 
@@ -5135,6 +5132,7 @@ def flatnonzero(a):
     return nonzero(ravel(a))[0]
 
 
+@set_module('mxnet.ndarray.numpy')
 def diag_indices_from(arr):
     """
     This returns a tuple of indices that can be used to access the main diagonal of an array
@@ -5171,7 +5169,7 @@ def diag_indices_from(arr):
         [  8,   9, 100,  11],
         [ 12,  13,  14, 100]])
     """
-    return tuple(_npi.diag_indices_from(arr))
+    return tuple(_api_internal.diag_indices_from(arr))
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7824,3 +7822,97 @@ def cumsum(a, axis=None, dtype=None, out=None):
            [ 4,  9, 15]])
     """
     return _api_internal.cumsum(a, axis, dtype, out)
+
+
+@set_module('mxnet.ndarray.numpy')
+def diag(v, k=0):
+    """
+    Extracts a diagonal or constructs a diagonal array.
+    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.
+    - 2-D arrays: extracts the k-th Diagonal
+
+    Parameters
+    ----------
+    array : ndarray
+        The array to apply diag method.
+    k : offset
+        extracts or constructs kth diagonal given input array
+
+    Returns
+    ----------
+    out : ndarray
+    The extracted diagonal or constructed diagonal array.
+
+    Examples
+    --------
+    >>> x = np.arange(9).reshape((3,3))
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5],
+           [6, 7, 8]])
+    >>> np.diag(x)
+    array([0, 4, 8])
+    >>> np.diag(x, k=1)
+    array([1, 5])
+    >>> np.diag(x, k=-1)
+    array([3, 7])
+
+    >>> np.diag(np.diag(x))
+    array([[0, 0, 0],
+           [0, 4, 0],
+           [0, 0, 8]])
+    """
+    return _api_internal.diag(v, k)
+
+
+@set_module('mxnet.ndarray.numpy')
+def diagonal(a, offset=0, axis1=0, axis2=1):
+    """
+    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of
+    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and
+    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the
+    resulting array can be determined by removing axis1 and axis2 and appending an index to the
+    right equal to the size of the resulting diagonals.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input data from which diagonal are taken.
+    offset: int, Optional
+        Offset of the diagonal from the main diagonal
+    axis1: int, Optional
+        Axis to be used as the first axis of the 2-D sub-arrays
+    axis2: int, Optional
+        Axis to be used as the second axis of the 2-D sub-arrays
+
+    Returns
+    -------
+    out : ndarray
+        Output result
+
+    Raises
+    -------
+    ValueError:  If the dimension of a is less than 2.
+
+    Examples
+    --------
+    >>> a = np.arange(4).reshape(2,2)
+    >>> a
+    array([[0, 1],
+        [2, 3]])
+    >>> np.diagonal(a)
+    array([0, 3])
+    >>> np.diagonal(a, 1)
+    array([1])
+
+    >>> a = np.arange(8).reshape(2,2,2)
+    >>>a
+    array([[[0, 1],
+            [2, 3]],
+            [[4, 5],
+            [6, 7]]])
+    >>> np.diagonal(a, 0, 0, 1)
+    array([[0, 6],
+            [1, 7]])
+    """
+    return _api_internal.diagonal(a, offset, axis1, axis2)
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 72e367aa7894..39ec2c6ff9a2 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -69,7 +69,7 @@
            'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
-           'pad', 'cumsum']
+           'pad', 'cumsum', 'diag', 'diagonal']
 
 __all__ += fallback.__all__
 
@@ -9940,3 +9940,97 @@ def cumsum(a, axis=None, dtype=None, out=None):
     """
     return _mx_nd_np.cumsum(a, axis=axis, dtype=dtype, out=out)
 # pylint: enable=redefined-outer-name
+
+
+@set_module('mxnet.numpy')
+def diag(v, k=0):
+    """
+    Extracts a diagonal or constructs a diagonal array.
+    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.
+    - 2-D arrays: extracts the k-th Diagonal
+
+    Parameters
+    ----------
+    array : ndarray
+        The array to apply diag method.
+    k : offset
+        extracts or constructs kth diagonal given input array
+
+    Returns
+    ----------
+    out : ndarray
+    The extracted diagonal or constructed diagonal array.
+
+    Examples
+    --------
+    >>> x = np.arange(9).reshape((3,3))
+    >>> x
+    array([[0, 1, 2],
+           [3, 4, 5],
+           [6, 7, 8]])
+    >>> np.diag(x)
+    array([0, 4, 8])
+    >>> np.diag(x, k=1)
+    array([1, 5])
+    >>> np.diag(x, k=-1)
+    array([3, 7])
+
+    >>> np.diag(np.diag(x))
+    array([[0, 0, 0],
+           [0, 4, 0],
+           [0, 0, 8]])
+    """
+    return _mx_nd_np.diag(v, k=k)
+
+
+@set_module('mxnet.numpy')
+def diagonal(a, offset=0, axis1=0, axis2=1):
+    """
+    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of
+    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and
+    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the
+    resulting array can be determined by removing axis1 and axis2 and appending an index to the
+    right equal to the size of the resulting diagonals.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input data from which diagonal are taken.
+    offset: int, Optional
+        Offset of the diagonal from the main diagonal
+    axis1: int, Optional
+        Axis to be used as the first axis of the 2-D sub-arrays
+    axis2: int, Optional
+        Axis to be used as the second axis of the 2-D sub-arrays
+
+    Returns
+    -------
+    out : ndarray
+        Output result
+
+    Raises
+    -------
+    ValueError:  If the dimension of a is less than 2.
+
+    Examples
+    --------
+    >>> a = np.arange(4).reshape(2,2)
+    >>> a
+    array([[0, 1],
+        [2, 3]])
+    >>> np.diagonal(a)
+    array([0, 3])
+    >>> np.diagonal(a, 1)
+    array([1])
+
+    >>> a = np.arange(8).reshape(2,2,2)
+    >>>a
+    array([[[0, 1],
+            [2, 3]],
+            [[4, 5],
+            [6, 7]]])
+    >>> np.diagonal(a, 0, 0, 1)
+    array([[0, 6],
+            [1, 7]])
+    """
+    return _mx_nd_np.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index e9868861d83d..83c7e5429073 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -51,7 +51,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
-           'where', 'bincount', 'pad', 'cumsum']
+           'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.symbol.numpy')
@@ -6863,4 +6863,58 @@ def cumsum(a, axis=None, dtype=None, out=None):
     return _npi.cumsum(a, axis=axis, dtype=dtype, out=out)
 
 
+@set_module('mxnet.symbol.numpy')
+def diag(v, k=0):
+    """
+    Extracts a diagonal or constructs a diagonal array.
+    - 1-D arrays: constructs a 2-D array with the input as its diagonal, all other elements are zero.
+    - 2-D arrays: extracts the k-th Diagonal
+
+    Parameters
+    ----------
+    array : _Symbol
+        The array to apply diag method.
+    k : offset
+        extracts or constructs kth diagonal given input array
+
+    Returns
+    ----------
+    out : _Symbol
+    The extracted diagonal or constructed diagonal array.
+    """
+    return _npi.diag(v, k=k)
+
+
+@set_module('mxnet.symbol.numpy')
+def diagonal(a, offset=0, axis1=0, axis2=1):
+    """
+    If a is 2-D, returns the diagonal of a with the given offset, i.e., the collection of elements of
+    the form a[i, i+offset]. If a has more than two dimensions, then the axes specified by axis1 and
+    axis2 are used to determine the 2-D sub-array whose diagonal is returned. The shape of the
+    resulting array can be determined by removing axis1 and axis2 and appending an index to the
+    right equal to the size of the resulting diagonals.
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input data from which diagonal are taken.
+    offset: int, Optional
+        Offset of the diagonal from the main diagonal
+    axis1: int, Optional
+        Axis to be used as the first axis of the 2-D sub-arrays
+    axis2: int, Optional
+        Axis to be used as the second axis of the 2-D sub-arrays
+
+    Returns
+    -------
+    out : _Symbol
+        Output result
+
+    Raises
+    -------
+    ValueError:  If the dimension of a is less than 2.
+    """
+    return _npi.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)
+
+
 _set_np_symbol_class(_Symbol)
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index 36d06c7fc4cc..ae8421ac4010 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -142,4 +142,53 @@ MXNET_REGISTER_API("_npi.rot90")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.diag")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_diag");
+  nnvm::NodeAttrs attrs;
+  op::NumpyDiagParam param;
+  param.k = args[1].operator int();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::NumpyDiagParam>(&attrs);
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.diagonal")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_diagonal");
+  nnvm::NodeAttrs attrs;
+  op::NumpyDiagonalParam param;
+  param.offset = args[1].operator int();
+  param.axis1 = args[2].operator int();
+  param.axis2 = args[3].operator int();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::NumpyDiagonalParam>(&attrs);
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.diag_indices_from")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_diag_indices_from");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index 0bbe263cfc76..2e48596cee9c 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -983,6 +983,11 @@ struct NumpyDiagParam : public dmlc::Parameter<NumpyDiagParam> {
                 "Use k>0 for diagonals above the main diagonal, "
                 "and k<0 for diagonals below the main diagonal. ");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream k_s;
+    k_s << k;
+    (*dict)["k"] = k_s.str();
+  }
 };
 
 inline mxnet::TShape NumpyDiagShapeImpl(const mxnet::TShape &ishape,
@@ -1006,7 +1011,7 @@ inline mxnet::TShape NumpyDiagShapeImpl(const mxnet::TShape &ishape,
   auto s = std::max(std::min(h, w), a);
   // s is the length of diagonal with k as the offset
 
-  int32_t n_dim = ishape.ndim() - 1;
+  int n_dim = ishape.ndim() - 1;
   mxnet::TShape oshape(n_dim, -1);
   oshape[n_dim - 1] = s;
   return oshape;
@@ -1177,8 +1182,8 @@ void NumpyDiagOpBackward(const nnvm::NodeAttrs &attrs,
 
 struct NumpyDiagonalParam : public dmlc::Parameter<NumpyDiagonalParam> {
   int offset;
-  int32_t axis1;
-  int32_t axis2;
+  int axis1;
+  int axis2;
   DMLC_DECLARE_PARAMETER(NumpyDiagonalParam) {
     DMLC_DECLARE_FIELD(offset)
       .set_default(0)
@@ -1195,12 +1200,21 @@ struct NumpyDiagonalParam : public dmlc::Parameter<NumpyDiagonalParam> {
       .describe("The second axis of the sub-arrays of interest. "
                 "Ignored when the input is a 1-D array.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream offset_s, axis1_s, axis2_s;
+    offset_s << offset;
+    axis1_s << axis1;
+    axis2_s << axis2;
+    (*dict)["offset"] = offset_s.str();
+    (*dict)["axis1"] = axis1_s.str();
+    (*dict)["axis2"] = axis2_s.str();
+  }
 };
 
 inline mxnet::TShape NumpyDiagonalShapeImpl(const mxnet::TShape& ishape, const int k,
-                                            const int32_t axis1, const int32_t axis2) {
-  int32_t x1 = CheckAxis(axis1, ishape.ndim());
-  int32_t x2 = CheckAxis(axis2, ishape.ndim());
+                                            const int axis1, const int axis2) {
+  int x1 = CheckAxis(axis1, ishape.ndim());
+  int x2 = CheckAxis(axis2, ishape.ndim());
 
   CHECK_NE(x1, x2) << "axis1 and axis2 cannot refer to the same axis " << x1;
 
@@ -1215,11 +1229,11 @@ inline mxnet::TShape NumpyDiagonalShapeImpl(const mxnet::TShape& ishape, const i
   if (s < 0) s = 0;
   if (x1 > x2) std::swap(x1, x2);
 
-  int32_t n_dim = ishape.ndim() - 1;
+  int n_dim = ishape.ndim() - 1;
   mxnet::TShape oshape(n_dim, -1);
 
   // remove axis1 and axis2 and append the new axis to the end
-  uint32_t idx = 0;
+  int idx = 0;
   for (int i = 0; i <= n_dim; ++i) {
     if (i != x1 && i != x2) {
       oshape[idx++] = ishape[i];
@@ -1292,22 +1306,22 @@ void NumpyDiagonalOpImpl(const TBlob& in_data,
                          const std::vector<OpReqType>& req) {
   using namespace mxnet_op;
   using namespace mshadow;
-  uint32_t x1 = CheckAxis(param.axis1, ishape.ndim());
-  uint32_t x2 = CheckAxis(param.axis2, ishape.ndim());
-  uint32_t idim = ishape.ndim(), odim = oshape.ndim();
-  uint32_t minx = x1, maxx = x2;
+  int x1 = CheckAxis(param.axis1, ishape.ndim());
+  int x2 = CheckAxis(param.axis2, ishape.ndim());
+  int idim = ishape.ndim(), odim = oshape.ndim();
+  int minx = x1, maxx = x2;
   if (minx > maxx) std::swap(minx, maxx);
 
   index_t oleading = 1,
           obody = 1,
           otrailing = 1;
-  for (uint32_t i = 0; i < minx; ++i) {
+  for (int i = 0; i < minx; ++i) {
     oleading *= ishape[i];
   }
-  for (uint32_t i = minx + 1; i < maxx; ++i) {
+  for (int i = minx + 1; i < maxx; ++i) {
     obody *= ishape[i];
   }
-  for (uint32_t i = maxx + 1; i < idim; ++i) {
+  for (int i = maxx + 1; i < idim; ++i) {
     otrailing *= ishape[i];
   }
 
diff --git a/src/operator/numpy/np_matrix_op.cc b/src/operator/numpy/np_matrix_op.cc
index e9d269dd54d6..1c0a8a610a6e 100644
--- a/src/operator/numpy/np_matrix_op.cc
+++ b/src/operator/numpy/np_matrix_op.cc
@@ -1435,7 +1435,7 @@ NNVM_REGISTER_OP(_npi_dsplit)
 .add_argument("data", "NDArray-or-Symbol", "The input")
 .add_arguments(SplitParam::__FIELDS__());
 
-NNVM_REGISTER_OP(_np_diag)
+NNVM_REGISTER_OP(_npi_diag)
 .set_attr_parser(ParamParser<NumpyDiagParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
@@ -1446,18 +1446,18 @@ NNVM_REGISTER_OP(_np_diag)
 .set_attr<mxnet::FInferShape>("FInferShape", NumpyDiagOpShape)
 .set_attr<nnvm::FInferType>("FInferType", NumpyDiagOpType)
 .set_attr<FCompute>("FCompute<cpu>", NumpyDiagOpForward<cpu>)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_diag"})
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_npi_diag"})
 .add_argument("data", "NDArray-or-Symbol", "Input ndarray")
 .add_arguments(NumpyDiagParam::__FIELDS__());
 
-NNVM_REGISTER_OP(_backward_np_diag)
+NNVM_REGISTER_OP(_backward_npi_diag)
 .set_attr_parser(ParamParser<NumpyDiagParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
 .set_attr<nnvm::TIsBackward>("TIsBackward", true)
 .set_attr<FCompute>("FCompute<cpu>", NumpyDiagOpBackward<cpu>);
 
-NNVM_REGISTER_OP(_np_diagonal)
+NNVM_REGISTER_OP(_npi_diagonal)
 .set_attr_parser(ParamParser<NumpyDiagonalParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
@@ -1468,11 +1468,11 @@ NNVM_REGISTER_OP(_np_diagonal)
 .set_attr<mxnet::FInferShape>("FInferShape", NumpyDiagonalOpShape)
 .set_attr<nnvm::FInferType>("FInferType", NumpyDiagonalOpType)
 .set_attr<FCompute>("FCompute<cpu>", NumpyDiagonalOpForward<cpu>)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_np_diagonal"})
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_npi_diagonal"})
 .add_argument("data", "NDArray-or-Symbol", "Input ndarray")
 .add_arguments(NumpyDiagonalParam::__FIELDS__());
 
-NNVM_REGISTER_OP(_backward_np_diagonal)
+NNVM_REGISTER_OP(_backward_npi_diagonal)
 .set_attr_parser(ParamParser<NumpyDiagonalParam>)
 .set_num_inputs(1)
 .set_num_outputs(1)
diff --git a/src/operator/numpy/np_matrix_op.cu b/src/operator/numpy/np_matrix_op.cu
index c9e896bc5b57..c4b3290d58b7 100644
--- a/src/operator/numpy/np_matrix_op.cu
+++ b/src/operator/numpy/np_matrix_op.cu
@@ -127,16 +127,16 @@ NNVM_REGISTER_OP(_npi_dsplit)
 NNVM_REGISTER_OP(_npx_reshape)
 .set_attr<FCompute>("FCompute<gpu>", UnaryOp::IdentityCompute<gpu>);
 
-NNVM_REGISTER_OP(_np_diag)
+NNVM_REGISTER_OP(_npi_diag)
 .set_attr<FCompute>("FCompute<gpu>", NumpyDiagOpForward<gpu>);
 
-NNVM_REGISTER_OP(_backward_np_diag)
+NNVM_REGISTER_OP(_backward_npi_diag)
 .set_attr<FCompute>("FCompute<gpu>", NumpyDiagOpBackward<gpu>);
 
-NNVM_REGISTER_OP(_np_diagonal)
+NNVM_REGISTER_OP(_npi_diagonal)
 .set_attr<FCompute>("FCompute<gpu>", NumpyDiagonalOpForward<gpu>);
 
-NNVM_REGISTER_OP(_backward_np_diagonal)
+NNVM_REGISTER_OP(_backward_npi_diagonal)
 .set_attr<FCompute>("FCompute<gpu>", NumpyDiagonalOpBackward<gpu>);
 
 NNVM_REGISTER_OP(_np_diagflat)

From 905eb2774403d24b5b738023a1d8b80790768adc Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Wed, 8 Apr 2020 08:02:26 +0800
Subject: [PATCH 18/44] ffi_atleast_1/2/3d (#17897)

---
 benchmark/python/ffi/benchmark_ffi.py |   3 +
 python/mxnet/_numpy_op_doc.py         | 107 -----------------------
 python/mxnet/ndarray/numpy/_op.py     | 118 ++++++++++++++++++++++++++
 python/mxnet/numpy/multiarray.py      | 112 ++++++++++++++++++++++++
 python/mxnet/symbol/numpy/_symbol.py  |  71 ++++++++++++++++
 src/api/operator/numpy/np_init_op.cc  |  81 ++++++++++++++++++
 src/operator/numpy/np_init_op.cc      |   2 +-
 src/operator/numpy/np_init_op.cu      |   6 +-
 src/operator/numpy/np_init_op.h       |   5 ++
 9 files changed, 394 insertions(+), 111 deletions(-)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index ac7793706a22..e4c56b6e57f1 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -61,6 +61,9 @@ def prepare_workloads():
     OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
     OpArgMngr.add_workload("argmax", pool['3x2'], axis=-1)
     OpArgMngr.add_workload("argmin", pool['3x2'], axis=-1)
+    OpArgMngr.add_workload("atleast_1d", pool['2'], pool['2x2'])
+    OpArgMngr.add_workload("atleast_2d", pool['2'], pool['2x2'])
+    OpArgMngr.add_workload("atleast_3d", pool['2'], pool['2x2'])
     OpArgMngr.add_workload("indices", dimensions=(1, 2, 3))
     OpArgMngr.add_workload("subtract", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("multiply", pool['2x2'], pool['2x2'])
diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 3d80ce03b44e..8341d43608ce 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -370,113 +370,6 @@ def _np_copy(a, out=None):
     pass
 
 
-def _np_atleast_1d(*arys):
-    """
-    Convert inputs to arrays with at least one dimension.
-
-    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.
-
-    Parameters
-    ----------
-    arys1, arys2, ... : ndarray
-        One or more input arrays.
-
-    Returns
-    -------
-    ret : ndarray
-        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.
-
-    See also
-    --------
-    atleast_2d, atleast_3d
-
-    Examples
-    --------
-    >>> np.atleast_1d(1.0)
-    array([1.])
-    >>> x = np.arange(9.0).reshape(3,3)
-    >>> np.atleast_1d(x)
-    array([[0., 1., 2.],
-           [3., 4., 5.],
-           [6., 7., 8.]])
-    >>> np.atleast_1d(np.array(1), np.array([3, 4]))
-    [array([1.]), array([3., 4.])]
-    """
-    pass
-
-
-def _np_atleast_2d(*arys):
-    """
-    Convert inputs to arrays with at least two dimensions.
-
-    Parameters
-    ----------
-    arys1, arys2, ... : ndarray
-        One or more input arrays.
-
-    Returns
-    -------
-    ret : ndarray
-        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.
-
-    See also
-    --------
-    atleast_1d, atleast_3d
-
-    Examples
-    --------
-    >>> np.atleast_2d(3.0)
-    array([[3.]])
-    >>> x = np.arange(3.0)
-    >>> np.atleast_2d(x)
-    array([[0., 1., 2.]])
-    >>> np.atleast_2d(np.array(1), np.array([1, 2]), np.array([[1, 2]]))
-    [array([[1.]]), array([[1., 2.]]), array([[1., 2.]])]
-    """
-    pass
-
-def _np_atleast_3d(*arys):
-    """
-    Convert inputs to arrays with at least three dimension.
-
-    Parameters
-    ----------
-    arys1, arys2, ... : ndarray
-        One or more input arrays.
-
-    Returns
-    -------
-    ret : ndarray
-        An array, or list of arrays, each with a.ndim >= 3.
-        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),
-        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).
-
-    See also
-    --------
-    atleast_1d, atleast_2d
-
-    Examples
-    --------
-    >>> np.atleast_3d(3.0)
-    array([[[3.]]])
-    >>> x = np.arange(3.0)
-    >>> np.atleast_3d(x).shape
-    (1, 3, 1)
-    >>> x = np.arange(12.0).reshape(4,3)
-    >>> np.atleast_3d(x).shape
-    (4, 3, 1)
-    >>> for arr in np.atleast_3d(np.array([1, 2]), np.array([[1, 2]]), np.array([[[1, 2]]])):
-    ...     print(arr, arr.shape)
-    ...
-    [[[1.]
-      [2.]]] (1, 2, 1)
-    [[[1.]
-      [2.]]] (1, 2, 1)
-    [[[1. 2.]]] (1, 1, 2)
-    """
-    pass
-
-
 def _np_reshape(a, newshape, order='C', out=None):
     """
     Gives a new shape to an array without changing its data.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index fdfefdd6793a..e32e16d2166c 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -46,6 +46,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
+           'atleast_1d', 'atleast_2d', 'atleast_3d',
            'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
@@ -7447,6 +7448,123 @@ def isfinite(x, out=None, **kwargs):
     return _unary_func_helper(x, _npi.isfinite, _np.isfinite, out=out, **kwargs)
 
 
+@set_module('mxnet.ndarray.numpy')
+def atleast_1d(*arys):
+    """
+    Convert inputs to arrays with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_2d, atleast_3d
+
+    Examples
+    --------
+    >>> np.atleast_1d(1.0)
+    array([1.])
+    >>> x = np.arange(9.0).reshape(3,3)
+    >>> np.atleast_1d(x)
+    array([[0., 1., 2.],
+           [3., 4., 5.],
+           [6., 7., 8.]])
+    >>> np.atleast_1d(np.array(1), np.array([3, 4]))
+    [array([1.]), array([3., 4.])]
+    """
+    if len(arys) == 1:
+        return _api_internal.atleast_1d(*arys)[0]
+    return list(_api_internal.atleast_1d(*arys))
+
+
+@set_module('mxnet.ndarray.numpy')
+def atleast_2d(*arys):
+    """
+    Convert inputs to arrays with at least two dimensions.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_1d, atleast_3d
+
+    Examples
+    --------
+    >>> np.atleast_2d(3.0)
+    array([[3.]])
+    >>> x = np.arange(3.0)
+    >>> np.atleast_2d(x)
+    array([[0., 1., 2.]])
+    >>> np.atleast_2d(np.array(1), np.array([1, 2]), np.array([[1, 2]]))
+    [array([[1.]]), array([[1., 2.]]), array([[1., 2.]])]
+    """
+    if len(arys) == 1:
+        return _api_internal.atleast_2d(*arys)[0]
+    return list(_api_internal.atleast_2d(*arys))
+
+
+@set_module('mxnet.ndarray.numpy')
+def atleast_3d(*arys):
+    """
+    Convert inputs to arrays with at least three dimension.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 3.
+        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),
+        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).
+
+    See also
+    --------
+    atleast_1d, atleast_2d
+
+    Examples
+    --------
+    >>> np.atleast_3d(3.0)
+    array([[[3.]]])
+    >>> x = np.arange(3.0)
+    >>> np.atleast_3d(x).shape
+    (1, 3, 1)
+    >>> x = np.arange(12.0).reshape(4,3)
+    >>> np.atleast_3d(x).shape
+    (4, 3, 1)
+    >>> for arr in np.atleast_3d(np.array([1, 2]), np.array([[1, 2]]), np.array([[[1, 2]]])):
+    ...     print(arr, arr.shape)
+    ...
+    [[[1.]
+      [2.]]] (1, 2, 1)
+    [[[1.]
+      [2.]]] (1, 2, 1)
+    [[[1. 2.]]] (1, 1, 2)
+    """
+    if len(arys) == 1:
+        return _api_internal.atleast_3d(*arys)[0]
+    return list(_api_internal.atleast_3d(*arys))
+
+
 @set_module('mxnet.ndarray.numpy')
 def where(condition, x=None, y=None):  # pylint: disable=too-many-return-statements
     """where(condition, [x, y])
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 39ec2c6ff9a2..c1ce3158c4fd 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -69,6 +69,7 @@
            'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
+           'atleast_1d', 'atleast_2d', 'atleast_3d',
            'pad', 'cumsum', 'diag', 'diagonal']
 
 __all__ += fallback.__all__
@@ -9789,6 +9790,117 @@ def bincount(x, weights=None, minlength=0):
     return _mx_nd_np.bincount(x, weights=weights, minlength=minlength)
 
 
+@set_module('mxnet.numpy')
+def atleast_1d(*arys):
+    """
+    Convert inputs to arrays with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_2d, atleast_3d
+
+    Examples
+    --------
+    >>> np.atleast_1d(1.0)
+    array([1.])
+    >>> x = np.arange(9.0).reshape(3,3)
+    >>> np.atleast_1d(x)
+    array([[0., 1., 2.],
+           [3., 4., 5.],
+           [6., 7., 8.]])
+    >>> np.atleast_1d(np.array(1), np.array([3, 4]))
+    [array([1.]), array([3., 4.])]
+    """
+    return _mx_nd_np.atleast_1d(*arys)
+
+
+@set_module('mxnet.numpy')
+def atleast_2d(*arys):
+    """
+    Convert inputs to arrays with at least two dimensions.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_1d, atleast_3d
+
+    Examples
+    --------
+    >>> np.atleast_2d(3.0)
+    array([[3.]])
+    >>> x = np.arange(3.0)
+    >>> np.atleast_2d(x)
+    array([[0., 1., 2.]])
+    >>> np.atleast_2d(np.array(1), np.array([1, 2]), np.array([[1, 2]]))
+    [array([[1.]]), array([[1., 2.]]), array([[1., 2.]])]
+    """
+    return _mx_nd_np.atleast_2d(*arys)
+
+
+@set_module('mxnet.numpy')
+def atleast_3d(*arys):
+    """
+    Convert inputs to arrays with at least three dimension.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : ndarray
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : ndarray
+        An array, or list of arrays, each with a.ndim >= 3.
+        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),
+        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).
+
+    See also
+    --------
+    atleast_1d, atleast_2d
+
+    Examples
+    --------
+    >>> np.atleast_3d(3.0)
+    array([[[3.]]])
+    >>> x = np.arange(3.0)
+    >>> np.atleast_3d(x).shape
+    (1, 3, 1)
+    >>> x = np.arange(12.0).reshape(4,3)
+    >>> np.atleast_3d(x).shape
+    (4, 3, 1)
+    >>> for arr in np.atleast_3d(np.array([1, 2]), np.array([[1, 2]]), np.array([[[1, 2]]])):
+    ...     print(arr, arr.shape)
+    ...
+    [[[1.]
+      [2.]]] (1, 2, 1)
+    [[[1.]
+      [2.]]] (1, 2, 1)
+    [[[1. 2.]]] (1, 1, 2)
+    """
+    return _mx_nd_np.atleast_3d(*arys)
+
+
 @set_module('mxnet.numpy')
 def pad(x, pad_width=None, mode="constant", **kwargs): # pylint: disable=too-many-arguments
     """
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 83c7e5429073..4588d86c4850 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -51,6 +51,7 @@
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
+           'atleast_1d', 'atleast_2d', 'atleast_3d',
            'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
@@ -6527,6 +6528,76 @@ def isfinite(x, out=None, **kwargs):
     return _unary_func_helper(x, _npi.isfinite, _np.isfinite, out=out, **kwargs)
 
 
+@set_module('mxnet.symbol.numpy')
+def atleast_1d(*arys):
+    """
+    Convert inputs to arrays with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional arrays, whilst higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : _Symbol
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : _Symbol
+        An array, or list of arrays, each with a.ndim >= 1. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_2d, atleast_3d
+    """
+    return _npi.atleast_1d(*arys)
+
+
+@set_module('mxnet.symbol.numpy')
+def atleast_2d(*arys):
+    """
+    Convert inputs to arrays with at least two dimensions.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : _Symbol
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : _Symbol
+        An array, or list of arrays, each with a.ndim >= 2. Copies are made only if necessary.
+
+    See also
+    --------
+    atleast_1d, atleast_3d
+    """
+    return _npi.atleast_2d(*arys)
+
+
+@set_module('mxnet.symbol.numpy')
+def atleast_3d(*arys):
+    """
+    Convert inputs to arrays with at least three dimension.
+
+    Parameters
+    ----------
+    arys1, arys2, ... : _Symbol
+        One or more input arrays.
+
+    Returns
+    -------
+    ret : _Symbol
+        An array, or list of arrays, each with a.ndim >= 3.
+        For example, a 1-D array of shape (N,) becomes a view of shape (1, N, 1),
+        and a 2-D array of shape (M, N) becomes a view of shape (M, N, 1).
+
+    See also
+    --------
+    atleast_1d, atleast_2d
+    """
+    return _npi.atleast_3d(*arys)
+
+
 @set_module('mxnet.symbol.numpy')
 def where(condition, x, y):
     """
diff --git a/src/api/operator/numpy/np_init_op.cc b/src/api/operator/numpy/np_init_op.cc
index c339fb505029..147f8a6b2022 100644
--- a/src/api/operator/numpy/np_init_op.cc
+++ b/src/api/operator/numpy/np_init_op.cc
@@ -120,4 +120,85 @@ MXNET_REGISTER_API("_npi.indices")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.atleast_1d")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_atleast_1d");
+  nnvm::NodeAttrs attrs;
+  op::AtleastNDParam param;
+  int args_size = args.size();
+  param.num_args = args_size;
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::AtleastNDParam>(&attrs);
+  int num_inputs = args_size;
+  std::vector<NDArray*> inputs_vec(args_size, nullptr);
+  for (int i = 0; i < args_size; ++i) {
+    inputs_vec[i] = args[i].operator mxnet::NDArray*();
+  }
+  NDArray** inputs = inputs_vec.data();
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+MXNET_REGISTER_API("_npi.atleast_2d")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_atleast_2d");
+  nnvm::NodeAttrs attrs;
+  op::AtleastNDParam param;
+  int args_size = args.size();
+  param.num_args = args_size;
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::AtleastNDParam>(&attrs);
+  int num_inputs = args_size;
+  std::vector<NDArray*> inputs_vec(args_size, nullptr);
+  for (int i = 0; i < args_size; ++i) {
+    inputs_vec[i] = args[i].operator mxnet::NDArray*();
+  }
+  NDArray** inputs = inputs_vec.data();
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+MXNET_REGISTER_API("_npi.atleast_3d")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_atleast_3d");
+  nnvm::NodeAttrs attrs;
+  op::AtleastNDParam param;
+  int args_size = args.size();
+  param.num_args = args_size;
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::AtleastNDParam>(&attrs);
+  int num_inputs = args_size;
+  std::vector<NDArray*> inputs_vec(args_size, nullptr);
+  for (int i = 0; i < args_size; ++i) {
+    inputs_vec[i] = args[i].operator mxnet::NDArray*();
+  }
+  NDArray** inputs = inputs_vec.data();
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
 }  // namespace mxnet
diff --git a/src/operator/numpy/np_init_op.cc b/src/operator/numpy/np_init_op.cc
index e6073bd2a22d..cffca8f10ba4 100644
--- a/src/operator/numpy/np_init_op.cc
+++ b/src/operator/numpy/np_init_op.cc
@@ -134,7 +134,7 @@ inline bool AtleastNDShape(const nnvm::NodeAttrs& attrs,
 }
 
 #define NNVM_REGISTER_ATLEAST_ND(N)                                       \
-NNVM_REGISTER_OP(_np_atleast_##N##d)                                      \
+NNVM_REGISTER_OP(_npi_atleast_##N##d)                                      \
 .set_attr_parser(ParamParser<AtleastNDParam>)                             \
 .set_num_inputs(                                                          \
 [](const NodeAttrs& attrs) {                                              \
diff --git a/src/operator/numpy/np_init_op.cu b/src/operator/numpy/np_init_op.cu
index 886bed61ec66..95e4322f31e7 100644
--- a/src/operator/numpy/np_init_op.cu
+++ b/src/operator/numpy/np_init_op.cu
@@ -41,13 +41,13 @@ NNVM_REGISTER_OP(_npi_identity)
 NNVM_REGISTER_OP(_npi_full_like)
 .set_attr<FCompute>("FCompute<gpu>", FullLikeOpCompute<gpu>);
 
-NNVM_REGISTER_OP(_np_atleast_1d)
+NNVM_REGISTER_OP(_npi_atleast_1d)
 .set_attr<FCompute>("FCompute<gpu>", AtleastNDCompute<gpu>);
 
-NNVM_REGISTER_OP(_np_atleast_2d)
+NNVM_REGISTER_OP(_npi_atleast_2d)
 .set_attr<FCompute>("FCompute<gpu>", AtleastNDCompute<gpu>);
 
-NNVM_REGISTER_OP(_np_atleast_3d)
+NNVM_REGISTER_OP(_npi_atleast_3d)
 .set_attr<FCompute>("FCompute<gpu>", AtleastNDCompute<gpu>);
 
 NNVM_REGISTER_OP(_npi_arange)
diff --git a/src/operator/numpy/np_init_op.h b/src/operator/numpy/np_init_op.h
index e92af5fc4544..021989eb5fce 100644
--- a/src/operator/numpy/np_init_op.h
+++ b/src/operator/numpy/np_init_op.h
@@ -287,6 +287,11 @@ struct AtleastNDParam : dmlc::Parameter<AtleastNDParam> {
     .set_lower_bound(1)
     .describe("Number of input arrays.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream num_args_s;
+    num_args_s << num_args;
+    (*dict)["num_args"] = num_args_s.str();
+  }
 };
 
 template<typename xpu>

From acb535b14854069f39ba28c7bfa410731e0b5693 Mon Sep 17 00:00:00 2001
From: Yijun Chen <chenyijun0902@gmail.com>
Date: Fri, 10 Apr 2020 14:01:24 +0800
Subject: [PATCH 19/44] add: numpy rollaxis (#17865)

---
 python/mxnet/ndarray/numpy/_op.py      | 34 ++++++++-
 python/mxnet/numpy/fallback.py         |  2 -
 python/mxnet/numpy/multiarray.py       | 39 +++++++++-
 python/mxnet/symbol/numpy/_symbol.py   | 37 +++++++++-
 src/operator/numpy/np_matrix_op-inl.h  | 98 ++++++++++++++++++++++++++
 src/operator/numpy/np_matrix_op.cc     | 64 +++++++++++++++++
 src/operator/numpy/np_matrix_op.cu     |  6 ++
 tests/python/unittest/test_numpy_op.py | 40 +++++++++++
 8 files changed, 314 insertions(+), 6 deletions(-)

diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index e32e16d2166c..fba28421c865 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -47,7 +47,7 @@
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
+           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -7942,6 +7942,38 @@ def cumsum(a, axis=None, dtype=None, out=None):
     return _api_internal.cumsum(a, axis, dtype, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+def rollaxis(a, axis, start=0):
+    """
+    Roll the specified axis backwards, until it lies in a given position.
+    a
+        Input array.
+    axis : integer
+        The axis to roll backwards. The positions of the other axes do not
+        change relative to one another.
+    start: int, optional
+        The axis is rolled until it lies before this position.
+        The default, 0, results in a “complete” roll.
+
+    Returns
+    -------
+    res : ndarray
+        A view after applying rollaxis to `a` is returned.
+
+    -----
+    Examples
+    --------
+    >>> a = np.ones((3,4,5,6))
+    >>> np.rollaxis(a, 3, 1).shape
+    (3, 6, 4, 5)
+    >>> np.rollaxis(a, 2).shape
+    (5, 3, 4, 6)
+    >>> np.rollaxis(a, 1, 4).shape
+    (3, 5, 6, 4)
+    """
+    return _npi.rollaxis(a, axis, start)
+
+
 @set_module('mxnet.ndarray.numpy')
 def diag(v, k=0):
     """
diff --git a/python/mxnet/numpy/fallback.py b/python/mxnet/numpy/fallback.py
index 54df715b60c0..7dddb100f2fa 100644
--- a/python/mxnet/numpy/fallback.py
+++ b/python/mxnet/numpy/fallback.py
@@ -90,7 +90,6 @@
     'rate',
     'real',
     'result_type',
-    'rollaxis',
     'roots',
     'searchsorted',
     'select',
@@ -182,7 +181,6 @@
 rate = onp.rate
 real = onp.real
 result_type = onp.result_type
-rollaxis = onp.rollaxis
 roots = onp.roots
 searchsorted = onp.searchsorted
 select = onp.select
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index c1ce3158c4fd..0f539f1ec8c6 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -70,7 +70,7 @@
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'pad', 'cumsum', 'diag', 'diagonal']
+           'pad', 'cumsum', 'rollaxis', 'diag', 'diagonal']
 
 __all__ += fallback.__all__
 
@@ -10051,7 +10051,42 @@ def cumsum(a, axis=None, dtype=None, out=None):
            [ 4,  9, 15]])
     """
     return _mx_nd_np.cumsum(a, axis=axis, dtype=dtype, out=out)
-# pylint: enable=redefined-outer-name
+
+
+# pylint: disable=redefined-outer-name
+@set_module('mxnet.numpy')
+def rollaxis(a, axis, start=0):
+    """
+    Roll the specified axis backwards, until it lies in a given position.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    axis : integer
+        The axis to roll backwards. The positions of the other axes do not
+        change relative to one another.
+    start: int, optional
+        The axis is rolled until it lies before this position.
+        The default, 0, results in a “complete” roll.
+
+    Returns
+    -------
+    res : ndarray
+        A view after applying rollaxis to `a` is returned.
+
+    -----
+    Examples
+    --------
+    >>> a = np.ones((3,4,5,6))
+    >>> np.rollaxis(a, 3, 1).shape
+    (3, 6, 4, 5)
+    >>> np.rollaxis(a, 2).shape
+    (5, 3, 4, 6)
+    >>> np.rollaxis(a, 1, 4).shape
+    (3, 5, 6, 4)
+    """
+    return _mx_nd_np.rollaxis(a, axis, start)
 
 
 @set_module('mxnet.numpy')
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 4588d86c4850..576c2a5341a3 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -52,7 +52,7 @@
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'where', 'bincount', 'pad', 'cumsum', 'diag', 'diagonal']
+           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.symbol.numpy')
@@ -6934,6 +6934,41 @@ def cumsum(a, axis=None, dtype=None, out=None):
     return _npi.cumsum(a, axis=axis, dtype=dtype, out=out)
 
 
+@set_module('mxnet.symbol.numpy')
+def rollaxis(a, axis, start=0):
+    """
+    Roll the specified axis backwards, until it lies in a given position.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input array.
+    axis : integer
+        The axis to roll backwards. The positions of the other axes do not
+        change relative to one another.
+    start: int, optional
+        The axis is rolled until it lies before this position.
+        The default, 0, results in a “complete” roll.
+
+    Returns
+    -------
+    res : ndarray
+        A view after applying rollaxis to `a` is returned.
+
+    -----
+    Examples
+    --------
+    >>> a = np.ones((3,4,5,6))
+    >>> np.rollaxis(a, 3, 1).shape
+    (3, 6, 4, 5)
+    >>> np.rollaxis(a, 2).shape
+    (5, 3, 4, 6)
+    >>> np.rollaxis(a, 1, 4).shape
+    (3, 5, 6, 4)
+    """
+    return _npi.rollaxis(a, axis, start)
+
+
 @set_module('mxnet.symbol.numpy')
 def diag(v, k=0):
     """
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index 2e48596cee9c..ed6bf27ac0f9 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -517,6 +517,47 @@ void NumpyFlipForward(const nnvm::NodeAttrs& attrs,
   NumpyFlipForwardImpl<xpu>(ctx, inputs, outputs, stride_, trailing_, flip_index);
 }
 
+struct NumpyRollaxisParam : public dmlc::Parameter<NumpyRollaxisParam> {
+  int axis;
+  int start;
+  DMLC_DECLARE_PARAMETER(NumpyRollaxisParam) {
+    DMLC_DECLARE_FIELD(axis)
+    .describe("The axis to roll backwards. "
+    "The positions of the other axes do not change relative to one another.");
+    DMLC_DECLARE_FIELD(start)
+    .set_default(0)
+    .describe("The axis is rolled until it lies before this position. "
+              "The default, 0, results in a “complete” roll.");
+  }
+};
+
+inline mxnet::TShape NumpyRollaxisShapeImpl(int axis,
+                                            int start,
+                                            const int& ndim) {
+  mxnet::TShape axes(ndim, -1);
+  if (axis < 0) {
+    axis += ndim;
+  }
+  if (start < 0) {
+    start += ndim;
+  }
+  if (axis < start) {
+    axes[start - 1] = axis;
+  } else {
+    axes[start] = axis;
+  }
+  int new_axis = 0;
+  for (int i = 0; i < axes.ndim(); i++) {
+    if (axes[i] < 0) {
+      if (new_axis == axis) {
+        new_axis++;
+      }
+      axes[i] = new_axis++;
+    }
+  }
+  return axes;
+}
+
 struct NumpyMoveaxisParam : public dmlc::Parameter<NumpyMoveaxisParam> {
   mxnet::TShape source;
   mxnet::TShape destination;
@@ -601,6 +642,63 @@ void NumpyMoveaxisCompute(const nnvm::NodeAttrs& attrs,
   })
 }
 
+template<typename xpu>
+void NumpyRollaxisCompute(const nnvm::NodeAttrs& attrs,
+                          const OpContext& ctx,
+                          const std::vector<TBlob>& inputs,
+                          const std::vector<OpReqType>& req,
+                          const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req[0], kWriteTo) << "Rollaxis does not support inplace";
+  mxnet::TShape axes;
+  const NumpyRollaxisParam& param = nnvm::get<NumpyRollaxisParam>(attrs.parsed);
+  axes = NumpyRollaxisShapeImpl(param.axis, param.start, inputs[0].ndim());
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, Dtype, {
+    TransposeImpl<xpu>(ctx.run_ctx, inputs[0], outputs[0], axes);
+  })
+}
+
+template<typename xpu>
+void NumpyRollaxisBackward(const nnvm::NodeAttrs &attrs,
+                            const OpContext &ctx,
+                            const std::vector<TBlob> &inputs,
+                            const std::vector<OpReqType> &req,
+                            const std::vector<TBlob> &outputs) {
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  const NumpyRollaxisParam& param = nnvm::get<NumpyRollaxisParam>(attrs.parsed);
+  int axis_origin = param.axis;
+  int start_origin = param.start;
+  int ndim = inputs[0].ndim();
+
+  int axis;
+  int start;
+
+  if (axis_origin < 0) {
+    axis_origin += ndim;
+  }
+
+  if (start_origin < 0) {
+    start_origin += ndim;
+  }
+
+  if (axis_origin < start_origin) {
+    axis = start_origin - 1;
+    start = axis_origin;
+  } else {
+    axis = start_origin;
+    start = axis_origin + 1;
+  }
+  mxnet::TShape axes;
+  axes = NumpyRollaxisShapeImpl(axis, start, inputs[0].ndim());
+  MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, Dtype, {
+    TransposeImpl<xpu>(ctx.run_ctx, inputs[0], outputs[0], axes);
+  })
+}
+
 struct NumpyRot90Param : public dmlc::Parameter<NumpyRot90Param> {
   int k;
   dmlc::optional<mxnet::TShape> axes;
diff --git a/src/operator/numpy/np_matrix_op.cc b/src/operator/numpy/np_matrix_op.cc
index 1c0a8a610a6e..9e2d91a31815 100644
--- a/src/operator/numpy/np_matrix_op.cc
+++ b/src/operator/numpy/np_matrix_op.cc
@@ -34,6 +34,7 @@ namespace op {
 DMLC_REGISTER_PARAMETER(NumpyTransposeParam);
 DMLC_REGISTER_PARAMETER(NumpyRollParam);
 DMLC_REGISTER_PARAMETER(NumpyMoveaxisParam);
+DMLC_REGISTER_PARAMETER(NumpyRollaxisParam);
 DMLC_REGISTER_PARAMETER(NumpyRot90Param);
 DMLC_REGISTER_PARAMETER(NumpyReshapeParam);
 DMLC_REGISTER_PARAMETER(NumpyXReshapeParam);
@@ -1190,6 +1191,69 @@ NNVM_REGISTER_OP(_npi_roll)
 .add_argument("data", "NDArray-or-Symbol", "Input ndarray")
 .add_arguments(NumpyRollParam::__FIELDS__());
 
+bool NumpyRollaxisShape(const nnvm::NodeAttrs& attrs,
+                        mxnet::ShapeVector *in_attrs,
+                        mxnet::ShapeVector *out_attrs) {
+  const NumpyRollaxisParam& param = nnvm::get<NumpyRollaxisParam>(attrs.parsed);
+  // check 1 input, 1 output
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  // check transpose dimentions no more than 6
+  mxnet::TShape& shp = (*in_attrs)[0];
+  CHECK_LE(shp.ndim(), 6) << "Transpose support at most 6 dimensions";
+
+  // check axis and start range
+  CHECK_GE(param.axis, -shp.ndim())
+  << "axis must be within the range of "
+  << -shp.ndim() << " and " << shp.ndim() - 1;
+  CHECK_LT(param.axis, shp.ndim())
+  << "axis must be within the range of "
+  << -shp.ndim() << " and " << shp.ndim() - 1;
+  CHECK_GE(param.start, -shp.ndim())
+  << "start must be within the range of "
+  << -shp.ndim() << " and " << shp.ndim();
+  CHECK_LE(param.start, shp.ndim())
+  << "start must be within the range of "
+  << -shp.ndim() << " and " << shp.ndim();
+
+  // generate output shape
+  mxnet::TShape ret(shp.ndim(), -1);
+  mxnet::TShape axes;
+
+  axes = NumpyRollaxisShapeImpl(param.axis, param.start, shp.ndim());
+  for (int i = 0; i < shp.ndim(); ++i) {
+    CHECK(axes[i] < static_cast<int64_t>(shp.ndim()));
+    ret[i] = shp[axes[i]];
+  }
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, ret);
+  return shape_is_known(ret);
+}
+
+NNVM_REGISTER_OP(_npi_rollaxis)
+.describe(R"code(Roll the specified axis backwards, 
+until it lies in a given position.)code" ADD_FILELINE)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyRollaxisParam>)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::string>{"data"};
+  })
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyRollaxisShape)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<FCompute>("FCompute<cpu>", NumpyRollaxisCompute<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_npi_rollaxis_backward"})
+.add_argument("data", "NDArray-or-Symbol", "Input ndarray")
+.add_arguments(NumpyRollaxisParam::__FIELDS__());
+
+NNVM_REGISTER_OP(_npi_rollaxis_backward)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyRollaxisParam>)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FCompute>("FCompute<cpu>", NumpyRollaxisBackward<cpu>);
+
 template<>
 void NumpyFlipForwardImpl<cpu>(const OpContext& ctx,
                                const std::vector<TBlob>& inputs,
diff --git a/src/operator/numpy/np_matrix_op.cu b/src/operator/numpy/np_matrix_op.cu
index c4b3290d58b7..4871629d98e5 100644
--- a/src/operator/numpy/np_matrix_op.cu
+++ b/src/operator/numpy/np_matrix_op.cu
@@ -112,6 +112,12 @@ NNVM_REGISTER_OP(_backward_npi_flip)
 NNVM_REGISTER_OP(_np_moveaxis)
 .set_attr<FCompute>("FCompute<gpu>", NumpyMoveaxisCompute<gpu>);
 
+NNVM_REGISTER_OP(_npi_rollaxis)
+.set_attr<FCompute>("FCompute<gpu>", NumpyRollaxisCompute<gpu>);
+
+NNVM_REGISTER_OP(_npi_rollaxis_backward)
+.set_attr<FCompute>("FCompute<gpu>", NumpyRollaxisBackward<gpu>);
+
 NNVM_REGISTER_OP(_npi_rot90)
 .set_attr<FCompute>("FCompute<gpu>", NumpyRot90Compute<gpu>);
 
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index cdba6f41b0df..24c7776b7a55 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -8521,6 +8521,46 @@ def hybrid_forward(self, F, x, *args, **kwargs):
         assert ret.asnumpy().shape == expected_ret.shape
 
 
+@with_seed()
+@use_np
+def test_np_rollaxis():
+    class TestRollaxis(HybridBlock):
+        def __init__(self, axis=0, start=0):
+            super(TestRollaxis, self).__init__()
+            self._axis = axis
+            self._start = start
+             
+        def hybrid_forward(self, F, a, *args, **kwargs):
+            return F.np.rollaxis(a, axis=self._axis, start=self._start)
+
+    dtypes = ['int32', 'int64', 'float16', 'float32', 'float64']
+    for hybridize in [False, True]:
+        for dtype in dtypes:
+            for ndim in [0, 1, 2, 3, 4, 5, 6]:
+                shape = rand_shape_nd(ndim, dim=5, allow_zero_size=True)
+                np_data = _np.random.uniform(low=-100, high=100, size=shape).astype(dtype)
+                mx_data = np.array(np_data, dtype=dtype)
+                for axis in range(-ndim, ndim):
+                    for start in range(-ndim, ndim + 1):
+                        # test gluon
+                        test_rollaxis = TestRollaxis(axis, start)
+                        if hybridize:
+                            test_rollaxis.hybridize()
+                        np_out = _np.rollaxis(np_data, axis=axis, start=start)
+                        mx_data.attach_grad()
+                        with mx.autograd.record():
+                            mx_out = test_rollaxis(mx_data)
+                        assert mx_out.shape == np_out.shape
+                        mx_out.backward()
+                        assert same(mx_data.grad.shape, mx_data.shape)
+                        assert same(mx_data.grad.asnumpy(), _np.ones(shape))
+                        # test imperative
+                        np_out = _np.rollaxis(np_data, axis=axis, start=start)
+                        mx_out = np.rollaxis(mx_data, axis=axis, start=start)
+                        assert np_out.dtype == mx_out.dtype
+                        assert same(mx_out.asnumpy(), np_out)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()

From ce37bef8c4a491fb25fafe34b6516ffa5c6b074c Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Thu, 26 Mar 2020 01:31:18 +0800
Subject: [PATCH 20/44] * impl - FFI for np einsum (#17869)

* impl - FFI for np dstack

* impl - benchmark np_einsum np_dstack

* impl - FFI for np_unique

* impl - benchmark np_unique

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py  |  3 ++
 python/mxnet/ndarray/numpy/_op.py      | 11 ++--
 src/api/operator/numpy/np_einsum_op.cc | 71 ++++++++++++++++++++++++++
 src/api/operator/numpy/np_matrix_op.cc | 26 ++++++++++
 src/api/operator/numpy/np_unique_op.cc | 64 +++++++++++++++++++++++
 src/operator/nn/concat-inl.h           |  7 +++
 src/operator/numpy/np_einsum_op-inl.h  |  9 ++++
 src/operator/numpy/np_unique_op.h      | 11 ++++
 tests/python/unittest/test_numpy_op.py | 22 ++++++--
 9 files changed, 212 insertions(+), 12 deletions(-)
 create mode 100644 src/api/operator/numpy/np_einsum_op.cc
 create mode 100644 src/api/operator/numpy/np_unique_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index e4c56b6e57f1..749a322187bf 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -51,6 +51,9 @@ def generate_workloads():
 def prepare_workloads():
     pool = generate_workloads()
     OpArgMngr.add_workload("zeros", (2, 2))
+    OpArgMngr.add_workload("einsum", "ii", pool['2x2'], optimize=False)
+    OpArgMngr.add_workload("unique", pool['1'], return_index=True, return_inverse=True, return_counts=True, axis=-1)
+    OpArgMngr.add_workload("dstack", (pool['2x1'], pool['2x1'], pool['2x1'], pool['2x1']))
     OpArgMngr.add_workload("polyval", dnp.arange(10), pool['2x2'])
     OpArgMngr.add_workload("ediff1d", pool['2x2'], pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("nan_to_num", pool['2x2'])
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index fba28421c865..61cc6b4a8f7e 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -951,11 +951,8 @@ def unique(ar, return_index=False, return_inverse=False, return_counts=False, ax
     >>> u[indices]
     array([1., 2., 6., 4., 2., 3., 2.])
     """
-    ret = _npi.unique(ar, return_index, return_inverse, return_counts, axis)
-    if isinstance(ret, list):
-        return tuple(ret)
-    else:
-        return ret
+    ret = list(_api_internal.unique(ar, return_index, return_inverse, return_counts, axis))
+    return ret[0] if len(ret) == 1 else tuple(ret)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4345,7 +4342,7 @@ def dstack(arrays):
            [[2, 3]],
            [[3, 4]]])
     """
-    return _npi.dstack(*arrays)
+    return _api_internal.dstack(*arrays)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6656,7 +6653,7 @@ def einsum(*operands, **kwargs):
 
     subscripts = operands[0]
     operands = operands[1:]
-    return _npi.einsum(*operands, subscripts=subscripts, out=out, optimize=int(optimize_arg))
+    return _api_internal.einsum(*operands, subscripts, out, int(optimize_arg))
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/src/api/operator/numpy/np_einsum_op.cc b/src/api/operator/numpy/np_einsum_op.cc
new file mode 100644
index 000000000000..a5b8339a619e
--- /dev/null
+++ b/src/api/operator/numpy/np_einsum_op.cc
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_einsum_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_einsum_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/np_einsum_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.einsum")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_einsum");
+  nnvm::NodeAttrs attrs;
+  op::NumpyEinsumParam param;
+  int args_size = args.size();
+  // param.num_args
+  param.num_args = args_size - 3;
+  // param.subscripts
+  param.subscripts = args[args_size - 3].operator std::string();
+  // param.optimize
+  param.optimize = args[args_size - 1].operator int();
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyEinsumParam>(&attrs);
+
+  // inputs
+  int num_inputs = param.num_args;
+  std::vector<NDArray*> inputs_vec(num_inputs, nullptr);
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs_vec[i] = args[i].operator mxnet::NDArray*();
+  }
+  NDArray** inputs = inputs_vec.data();
+
+  // outputs
+  NDArray* out = args[args_size - 2].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(args_size - 2);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index ae8421ac4010..fdf8e9a081fa 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -24,6 +24,7 @@
 #include <mxnet/api_registry.h>
 #include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
+#include "../../../operator/nn/concat-inl.h"
 #include "../../../operator/tensor/matrix_op-inl.h"
 #include "../../../operator/numpy/np_matrix_op-inl.h"
 
@@ -49,6 +50,31 @@ MXNET_REGISTER_API("_npi.expand_dims")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.dstack")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_dstack");
+  nnvm::NodeAttrs attrs;
+  op::ConcatParam param;
+  int args_size = args.size();
+  // param.num_args
+  param.num_args = args_size;
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::ConcatParam>(&attrs);
+  // inputs
+  int num_inputs = args_size;
+  std::vector<NDArray*> inputs_vec(args_size, nullptr);
+  for (int i = 0; i < args_size; ++i) {
+    inputs_vec[i] = args[i].operator mxnet::NDArray*();
+  }
+  NDArray** inputs = inputs_vec.data();
+  // outputs
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
 MXNET_REGISTER_API("_npi.split")
 .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
   using namespace runtime;
diff --git a/src/api/operator/numpy/np_unique_op.cc b/src/api/operator/numpy/np_unique_op.cc
new file mode 100644
index 000000000000..288260f5dfb2
--- /dev/null
+++ b/src/api/operator/numpy/np_unique_op.cc
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_unique_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_unique_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/np_unique_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.unique")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_unique");
+  nnvm::NodeAttrs attrs;
+  op::NumpyUniqueParam param;
+  // param
+  param.return_index = args[1].operator bool();
+  param.return_inverse = args[2].operator bool();
+  param.return_counts = args[3].operator bool();
+  if (args[4].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[4].operator int();
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyUniqueParam>(&attrs);
+  // inputs
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  // outputs
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+}  // namespace mxnet
diff --git a/src/operator/nn/concat-inl.h b/src/operator/nn/concat-inl.h
index ffedba46c1ac..b5505d12ca45 100644
--- a/src/operator/nn/concat-inl.h
+++ b/src/operator/nn/concat-inl.h
@@ -55,6 +55,13 @@ struct ConcatParam : public dmlc::Parameter<ConcatParam> {
     DMLC_DECLARE_FIELD(dim).set_default(1)
     .describe("the dimension to be concated.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream num_args_s, dim_s;
+    num_args_s << num_args;
+    dim_s << dim;
+    (*dict)["num_args"] = num_args_s.str();
+    (*dict)["dim"] = dim_s.str();
+  }
 };  // struct ConcatParam
 
 template<typename xpu, typename DType>
diff --git a/src/operator/numpy/np_einsum_op-inl.h b/src/operator/numpy/np_einsum_op-inl.h
index 0a964e279bc7..05dc5209cc36 100644
--- a/src/operator/numpy/np_einsum_op-inl.h
+++ b/src/operator/numpy/np_einsum_op-inl.h
@@ -403,6 +403,15 @@ struct NumpyEinsumParam: public dmlc::Parameter<NumpyEinsumParam> {
     DMLC_DECLARE_FIELD(optimize)
       .set_default(0);
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream num_args_s, optimize_s, subscripts_s;
+    num_args_s << num_args;
+    optimize_s << optimize;
+    subscripts_s << subscripts;
+    (*dict)["num_args"] = num_args_s.str();
+    (*dict)["optimize"] = optimize_s.str();
+    (*dict)["subscripts"] = subscripts_s.str();
+  }
 };
 
 class EinsumOp {
diff --git a/src/operator/numpy/np_unique_op.h b/src/operator/numpy/np_unique_op.h
index bc2b6c34c19f..0a121cd69481 100644
--- a/src/operator/numpy/np_unique_op.h
+++ b/src/operator/numpy/np_unique_op.h
@@ -80,6 +80,17 @@ struct NumpyUniqueParam : public dmlc::Parameter<NumpyUniqueParam> {
     .set_default(dmlc::optional<int>())
     .describe("An integer that represents the axis to operator on.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream return_index_s, return_inverse_s, return_counts_s, axis_s;
+    return_index_s << return_index;
+    return_inverse_s << return_inverse;
+    return_counts_s << return_counts;
+    axis_s << axis;
+    (*dict)["return_index"] = return_index_s.str();
+    (*dict)["return_inverse"] = return_inverse_s.str();
+    (*dict)["return_counts"] = return_counts_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 }  // namespace op
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 24c7776b7a55..241afd858c02 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -6577,6 +6577,9 @@ def hybrid_forward(self, F, a):
     configs = [
         ((), True, True, True, None),
         ((1, ), True, True, True, -1),
+        ((5, ), False, False, False, 0),
+        ((5, ), True, False, False, 0),
+        ((5, ), True, True, False, 0),
         ((5, ), True, True, True, 0),
         ((5, ), True, True, True, None),
         ((5, 4), True, True, True, None),
@@ -6597,15 +6600,24 @@ def hybrid_forward(self, F, a):
                 x = np.array(x, dtype=dtype)
                 np_out = _np.unique(x.asnumpy(), *config[1:])
                 mx_out = test_unique(x)
-                assert mx_out[0].shape == np_out[0].shape
-                for i in range(4):
-                    assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5)
+                if (len(mx_out)) == 1:
+                    assert mx_out.shape == np_out.shape
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+                else:
+                    for i in range(len(mx_out)):
+                        assert mx_out[i].shape == np_out[i].shape
+                        assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5)
 
                 # Test imperative once again
                 mx_out = np.unique(x, *config[1:])
                 np_out = _np.unique(x.asnumpy(), *config[1:])
-                for i in range(4):
-                    assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5)
+                if (len(mx_out)) == 1:
+                    assert mx_out.shape == np_out.shape
+                    assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5)
+                else:
+                    for i in range(len(mx_out)):
+                        assert mx_out[i].shape == np_out[i].shape
+                        assert_almost_equal(mx_out[i].asnumpy(), np_out[i], rtol=1e-3, atol=1e-5)
 
 
 @with_seed()

From 28dbfda1dfe4f3fb1715aab33f8a2f1ca66be39c Mon Sep 17 00:00:00 2001
From: Yiyan66 <57363390+Yiyan66@users.noreply.github.com>
Date: Thu, 12 Mar 2020 16:47:53 +0800
Subject: [PATCH 21/44] [numpy] add op random.f (#17586)

* F

* f
---
 python/mxnet/ndarray/numpy/random.py   | 66 +++++++++++++++++++++++++-
 python/mxnet/numpy/random.py           | 64 ++++++++++++++++++++++++-
 python/mxnet/symbol/numpy/random.py    | 39 ++++++++++++++-
 tests/python/unittest/test_numpy_op.py | 31 ++++++++++++
 4 files changed, 197 insertions(+), 3 deletions(-)

diff --git a/python/mxnet/ndarray/numpy/random.py b/python/mxnet/ndarray/numpy/random.py
index 76cf3a5ea4ad..ff1f7fde106e 100644
--- a/python/mxnet/ndarray/numpy/random.py
+++ b/python/mxnet/ndarray/numpy/random.py
@@ -23,7 +23,7 @@
 
 
 __all__ = ['randint', 'uniform', 'normal', "choice", "rand", "multinomial", "multivariate_normal",
-           'logistic', 'gumbel', "rayleigh",
+           'logistic', 'gumbel', "rayleigh", 'f',
            "shuffle", 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal', 'weibull', 'pareto', 'power']
 
 
@@ -865,6 +865,70 @@ def beta(a, b, size=None, dtype=None, ctx=None):
     return out.astype(dtype)
 
 
+def f(dfnum, dfden, size=None, ctx=None):
+    r"""Draw samples from an F distribution.
+
+    Samples are drawn from an F distribution with specified parameters,
+    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of
+    freedom in denominator), where both parameters must be greater than
+    zero.
+
+    The random variate of the F distribution (also known as the
+    Fisher distribution) is a continuous probability distribution
+    that arises in ANOVA tests, and is the ratio of two chi-square
+    variates.
+
+    Parameters
+    ----------
+    dfnum : float or ndarray of floats
+        Degrees of freedom in numerator, must be > 0.
+    dfden : float or ndarray of float
+        Degrees of freedom in denominator, must be > 0.
+    size : int or tuple of ints, optional
+        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+        ``m * n * k`` samples are drawn.  If size is ``None`` (default),
+        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.
+        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.
+    ctx : Context, optional
+        Device context of output. Default is current context.
+
+    Returns
+    -------
+    out : ndarray or scalar
+        Drawn samples from the parameterized Fisher distribution.
+
+    Examples
+    --------
+    An example from Glantz[1], pp 47-40:
+
+    Two groups, children of diabetics (25 people) and children from people
+    without diabetes (25 controls). Fasting blood glucose was measured,
+    case group had a mean value of 86.1, controls had a mean value of
+    82.2. Standard deviations were 2.09 and 2.49 respectively. Are these
+    data consistent with the null hypothesis that the parents diabetic
+    status does not affect their children's blood glucose levels?
+    Calculating the F statistic from the data gives a value of 36.01.
+
+    Draw samples from the distribution:
+
+    >>> dfnum = 1. # between group degrees of freedom
+    >>> dfden = 48. # within groups degrees of freedom
+    >>> s = np.random.f(dfnum, dfden, 1000)
+
+    The lower bound for the top 1% of the samples is :
+
+    >>> np.sort(s)[-10]
+    7.61988120985 # random
+
+    So there is about a 1% chance that the F statistic will exceed 7.62,
+    the measured value is 36, so the null hypothesis is rejected at the 1%
+    level.
+    """
+    X = chisquare(df=dfnum, size=size, ctx=ctx)
+    Y = chisquare(df=dfden, size=size, ctx=ctx)
+    return (X * dfden) / (Y * dfnum)
+
+
 def chisquare(df, size=None, dtype=None, ctx=None):
     r"""
     chisquare(df, size=None, dtype=None, ctx=None)
diff --git a/python/mxnet/numpy/random.py b/python/mxnet/numpy/random.py
index f272ecbbbb4d..b07cc4ab37b7 100644
--- a/python/mxnet/numpy/random.py
+++ b/python/mxnet/numpy/random.py
@@ -21,7 +21,7 @@
 
 
 __all__ = ["randint", "uniform", "normal", "choice", "rand", "multinomial", "multivariate_normal",
-           "logistic", "gumbel",
+           "logistic", "gumbel", "f",
            "shuffle", "randn", "gamma", "beta", "chisquare", "exponential", "lognormal",
            "weibull", "pareto", "power", "rayleigh"]
 
@@ -873,6 +873,68 @@ def beta(a, b, size=None, dtype=None, ctx=None):
     return _mx_nd_np.random.beta(a, b, size=size, dtype=dtype, ctx=ctx)
 
 
+def f(dfnum, dfden, size=None, ctx=None):
+    r"""Draw samples from an F distribution.
+
+    Samples are drawn from an F distribution with specified parameters,
+    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of
+    freedom in denominator), where both parameters must be greater than
+    zero.
+
+    The random variate of the F distribution (also known as the
+    Fisher distribution) is a continuous probability distribution
+    that arises in ANOVA tests, and is the ratio of two chi-square
+    variates.
+
+    Parameters
+    ----------
+    dfnum : float or ndarray of floats
+        Degrees of freedom in numerator, must be > 0.
+    dfden : float or ndarray of float
+        Degrees of freedom in denominator, must be > 0.
+    size : int or tuple of ints, optional
+        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+        ``m * n * k`` samples are drawn.  If size is ``None`` (default),
+        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.
+        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.
+    ctx : Context, optional
+        Device context of output. Default is current context.
+
+    Returns
+    -------
+    out : ndarray or scalar
+        Drawn samples from the parameterized Fisher distribution.
+
+    Examples
+    --------
+    An example from Glantz[1], pp 47-40:
+
+    Two groups, children of diabetics (25 people) and children from people
+    without diabetes (25 controls). Fasting blood glucose was measured,
+    case group had a mean value of 86.1, controls had a mean value of
+    82.2. Standard deviations were 2.09 and 2.49 respectively. Are these
+    data consistent with the null hypothesis that the parents diabetic
+    status does not affect their children's blood glucose levels?
+    Calculating the F statistic from the data gives a value of 36.01.
+
+    Draw samples from the distribution:
+
+    >>> dfnum = 1. # between group degrees of freedom
+    >>> dfden = 48. # within groups degrees of freedom
+    >>> s = np.random.f(dfnum, dfden, 1000)
+
+    The lower bound for the top 1% of the samples is :
+
+    >>> np.sort(s)[-10]
+    7.61988120985 # random
+
+    So there is about a 1% chance that the F statistic will exceed 7.62,
+    the measured value is 36, so the null hypothesis is rejected at the 1%
+    level.
+    """
+    return _mx_nd_np.random.f(dfnum, dfden, size=size, ctx=ctx)
+
+
 def chisquare(df, size=None, dtype=None, ctx=None):
     r"""
     chisquare(df, size=None, dtype=None, ctx=None)
diff --git a/python/mxnet/symbol/numpy/random.py b/python/mxnet/symbol/numpy/random.py
index 7afe8a98636f..46be06969173 100644
--- a/python/mxnet/symbol/numpy/random.py
+++ b/python/mxnet/symbol/numpy/random.py
@@ -22,7 +22,7 @@
 
 
 __all__ = ['randint', 'uniform', 'normal', 'multivariate_normal',
-           'logistic', 'gumbel', 'rayleigh',
+           'logistic', 'gumbel', 'rayleigh', 'f',
            'rand', 'shuffle', 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal',
            'weibull', 'pareto', 'power']
 
@@ -568,6 +568,43 @@ def beta(a, b, size=None, dtype=None, ctx=None):
     return out.astype(dtype)
 
 
+def f(dfnum, dfden, size=None, ctx=None):
+    r"""Draw samples from an F distribution.
+
+    Samples are drawn from an F distribution with specified parameters,
+    `dfnum` (degrees of freedom in numerator) and `dfden` (degrees of
+    freedom in denominator), where both parameters must be greater than
+    zero.
+
+    The random variate of the F distribution (also known as the
+    Fisher distribution) is a continuous probability distribution
+    that arises in ANOVA tests, and is the ratio of two chi-square
+    variates.
+
+    Parameters
+    ----------
+    dfnum : float or _Symbol of floats
+        Degrees of freedom in numerator, must be > 0.
+    dfden : float or _Symbol of float
+        Degrees of freedom in denominator, must be > 0.
+    size : int or tuple of ints, optional
+        Output shape.  If the given shape is, e.g., ``(m, n, k)``, then
+        ``m * n * k`` samples are drawn.  If size is ``None`` (default),
+        a single value is returned if ``dfnum`` and ``dfden`` are both scalars.
+        Otherwise, ``np.broadcast(dfnum, dfden).size`` samples are drawn.
+    ctx : Context, optional
+        Device context of output. Default is current context.
+
+    Returns
+    -------
+    out : _Symbol
+        Drawn samples from the parameterized Fisher distribution.
+    """
+    X = chisquare(df=dfnum, size=size, ctx=ctx)
+    Y = chisquare(df=dfden, size=size, ctx=ctx)
+    return (X * dfden) / (Y * dfnum)
+
+
 def chisquare(df, size=None, dtype=None, ctx=None):
     r"""
     chisquare(df, size=None, dtype=None, ctx=None)
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 241afd858c02..6c281cc4a125 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -4071,6 +4071,37 @@ def _test_random_beta_range(output):
         assert _test_random_beta_range(mx_out_imperative.asnumpy()) == True
 
 
+@with_seed()
+@use_np
+def test_np_random_f():
+    class TestRandomF(HybridBlock):
+        def __init__(self, size=None):
+            super(TestRandomF, self).__init__()
+            self._size = size
+
+        def hybrid_forward(self, F, dfnum, dfden):
+            return F.np.random.f(dfnum, dfden, size=self._size)
+
+    shape_list = [(), (1,), (2, 3), (4, 0, 5), 6, (7, 8), None]
+    hybridize_list = [False, True]
+    df = np.array([1])
+    for [param_shape, hybridize] in itertools.product(shape_list,
+         hybridize_list):
+        if sys.version_info.major < 3 and param_shape == ():
+            continue
+        mx_df = df
+        np_df = mx_df.asnumpy()
+        test_random_f = TestRandomF(size=param_shape)
+        if hybridize:
+            test_random_f.hybridize()
+        np_out = _np.random.f(np_df, np_df, size=param_shape)
+        mx_out = test_random_f(mx_df, mx_df)
+        mx_out_imperative = mx.np.random.f(mx_df, mx_df, size=param_shape)
+
+        assert_almost_equal(np_out.shape, mx_out.shape)
+        assert_almost_equal(np_out.shape, mx_out_imperative.shape)
+
+
 @with_seed()
 @use_np
 def test_np_random_chisquare():

From 573830a2a121de84ef9d5ec96eb7e7907771bbd6 Mon Sep 17 00:00:00 2001
From: AntiZpvoh <59728467+AntiZpvoh@users.noreply.github.com>
Date: Mon, 16 Mar 2020 12:52:53 +0800
Subject: [PATCH 22/44] [NumPy] add op random.laplace (#17316)

* NumPy Laplace Distribution partly Frontend and Backend

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* NumPy Laplace Distribution Backend style rectified

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* NumPy Laplace Distribution Frontend modified

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* Laplece op nightly test and normal op test correction

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* NumPy Laplace Distribution unit test and code style

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* Register uniform_n in CUDA

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* Delete the registering of Laplace_n

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* fix some alignment and indentation problems

Signed-off-by: AntiZpvoh <wwwqyy@126.com>

* fix some sanity problems such as too long lines

* fix some sanity problems again

* laplace parmeters form change

* implement basic laplace function

* add frontend implement and ndarray loc case

* complete the frontend

* fix some sanity problems

* fix some sanity problems

* fix some typos

* fix some problems

* fix a typo

* add size==() condition handling

* fix some typos

* remove unused code

Co-authored-by: Ubuntu <ubuntu@ip-172-31-15-23.us-east-2.compute.internal>
---
 python/mxnet/ndarray/numpy/random.py          |  42 ++++
 python/mxnet/numpy/random.py                  |  31 +++
 python/mxnet/symbol/numpy/random.py           |  52 +++-
 .../operator/numpy/random/np_laplace_op.cc    |  96 ++++++++
 src/operator/numpy/random/np_laplace_op.cc    |  69 ++++++
 src/operator/numpy/random/np_laplace_op.cu    |  35 +++
 src/operator/numpy/random/np_laplace_op.h     | 231 ++++++++++++++++++
 tests/nightly/test_np_random.py               |  19 ++
 tests/python/unittest/test_numpy_op.py        |   2 +-
 9 files changed, 575 insertions(+), 2 deletions(-)
 create mode 100644 src/api/operator/numpy/random/np_laplace_op.cc
 create mode 100644 src/operator/numpy/random/np_laplace_op.cc
 create mode 100644 src/operator/numpy/random/np_laplace_op.cu
 create mode 100644 src/operator/numpy/random/np_laplace_op.h

diff --git a/python/mxnet/ndarray/numpy/random.py b/python/mxnet/ndarray/numpy/random.py
index ff1f7fde106e..2acb1612726d 100644
--- a/python/mxnet/ndarray/numpy/random.py
+++ b/python/mxnet/ndarray/numpy/random.py
@@ -20,10 +20,12 @@
 from ...context import current_context
 from . import _internal as _npi
 from ..ndarray import NDArray
+from . import _api_internal
 
 
 __all__ = ['randint', 'uniform', 'normal', "choice", "rand", "multinomial", "multivariate_normal",
            'logistic', 'gumbel', "rayleigh", 'f',
+           'laplace',
            "shuffle", 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal', 'weibull', 'pareto', 'power']
 
 
@@ -1067,3 +1069,43 @@ def shuffle(x):
            [0., 1., 2.]])
     """
     _npi.shuffle(x, out=x)
+
+
+def laplace(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):
+    r"""Draw random samples from a Laplace distribution.
+
+    Samples are distributed according to a Laplace distribution parametrized
+    by *loc* (mean) and *scale* (the exponential decay).
+
+
+    Parameters
+    ----------
+    loc : float, The position of the distribution peak.
+
+    scale : float, the exponential decay.
+
+    size : int or tuple of ints, optional. Output shape.
+        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.
+        Default is None, in which case a single value is returned.
+
+    dtype : {'float16', 'float32', 'float64'}, optional
+        Data type of output samples. Default is 'float32'
+    ctx : Context, optional
+        Device context of output. Default is current context.
+    out : ``ndarray``, optional
+        Store output to an existing ``ndarray``.
+
+    Returns
+    -------
+    out : ndarray
+        Drawn samples from the parameterized Laplace distribution.
+    """
+    if ctx is None:
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = np.dtype(dtype).name
+    if size == ():
+        size = None
+    return _api_internal.laplace(loc, scale, size, dtype, ctx, out)
diff --git a/python/mxnet/numpy/random.py b/python/mxnet/numpy/random.py
index b07cc4ab37b7..c6690f149fe8 100644
--- a/python/mxnet/numpy/random.py
+++ b/python/mxnet/numpy/random.py
@@ -22,6 +22,7 @@
 
 __all__ = ["randint", "uniform", "normal", "choice", "rand", "multinomial", "multivariate_normal",
            "logistic", "gumbel", "f",
+           "laplace",
            "shuffle", "randn", "gamma", "beta", "chisquare", "exponential", "lognormal",
            "weibull", "pareto", "power", "rayleigh"]
 
@@ -1043,3 +1044,33 @@ def randn(*size, **kwargs):
     for s in size:
         output_shape += (s,)
     return _mx_nd_np.random.normal(0, 1, size=output_shape, **kwargs)
+
+def laplace(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):
+    r"""Draw random samples from a Laplace distribution.
+
+    Samples are distributed according to a Laplace distribution parametrized
+    by *loc* (mean) and *scale* (the exponential decay).
+
+    Parameters
+    ----------
+    loc : float, The position of the distribution peak.
+
+    scale : float, the exponential decay.
+
+    size : int or tuple of ints, optional. Output shape.
+        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.
+        Default is None, in which case a single value is returned.
+
+    dtype : {'float16', 'float32', 'float64'}, optional
+        Data type of output samples. Default is 'float32'
+    ctx : Context, optional
+        Device context of output. Default is current context.
+    out : ``ndarray``, optional
+        Store output to an existing ``ndarray``.
+
+    Returns
+    -------
+    out : ndarray
+        Drawn samples from the parameterized Laplace distribution.
+    """
+    return _mx_nd_np.random.laplace(loc, scale, size, dtype, ctx, out)
diff --git a/python/mxnet/symbol/numpy/random.py b/python/mxnet/symbol/numpy/random.py
index 46be06969173..e6e729f19fe4 100644
--- a/python/mxnet/symbol/numpy/random.py
+++ b/python/mxnet/symbol/numpy/random.py
@@ -24,7 +24,7 @@
 __all__ = ['randint', 'uniform', 'normal', 'multivariate_normal',
            'logistic', 'gumbel', 'rayleigh', 'f',
            'rand', 'shuffle', 'gamma', 'beta', 'chisquare', 'exponential', 'lognormal',
-           'weibull', 'pareto', 'power']
+           'weibull', 'pareto', 'power', 'laplace']
 
 
 def randint(low, high=None, size=None, dtype=None, ctx=None, out=None):
@@ -416,6 +416,56 @@ def choice(a, size=None, replace=True, p=None, ctx=None, out=None):
             return _npi.choice(p, a=a, size=size, replace=replace, ctx=ctx, weighted=True, out=out)
 
 
+def laplace(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):
+    r"""Draw random samples from a Laplace distribution.
+
+    Samples are distributed according to a Laplace distribution parametrized
+    by *loc* (mean) and *scale* (the exponential decay).
+
+    Parameters
+    ----------
+    loc : float, The position of the distribution peak.
+
+    scale : float, the exponential decay.
+
+    size : int or tuple of ints, optional. Output shape.
+        If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.
+        Default is None, in which case a single value is returned.
+
+    dtype : {'float16', 'float32', 'float64'}, optional
+        Data type of output samples. Default is 'float32'
+    ctx : Context, optional
+        Device context of output. Default is current context.
+    out : ``ndarray``, optional
+        Store output to an existing ``ndarray``.
+
+    Returns
+    -------
+    out : _Symbol (symbol representing `mxnet.numpy.ndarray` in computational graphs)
+        Drawn samples from the parameterized Laplace distribution.
+    """
+    from ._symbol import _Symbol as np_symbol
+    input_type = (isinstance(loc, np_symbol), isinstance(scale, np_symbol))
+    if dtype is None:
+        dtype = 'float32'
+    if ctx is None:
+        ctx = current_context()
+    if size == ():
+        size = None
+    if input_type == (True, True):
+        return _npi.laplace(loc, scale, loc=None, scale=None, size=size,
+                            ctx=ctx, dtype=dtype, out=out)
+    elif input_type == (False, True):
+        return _npi.laplace(scale, loc=loc, scale=None, size=size,
+                            ctx=ctx, dtype=dtype, out=out)
+    elif input_type == (True, False):
+        return _npi.laplace(loc, loc=None, scale=scale, size=size,
+                            ctx=ctx, dtype=dtype, out=out)
+    else:
+        return _npi.laplace(loc=loc, scale=scale, size=size,
+                            ctx=ctx, dtype=dtype, out=out)
+
+
 def gamma(shape, scale=1.0, size=None, dtype=None, ctx=None, out=None):
     """Draw samples from a Gamma distribution.
 
diff --git a/src/api/operator/numpy/random/np_laplace_op.cc b/src/api/operator/numpy/random/np_laplace_op.cc
new file mode 100644
index 000000000000..40e79017c0f2
--- /dev/null
+++ b/src/api/operator/numpy/random/np_laplace_op.cc
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_laplace_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_laplace_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_laplace_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.laplace")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_laplace");
+  nnvm::NodeAttrs attrs;
+  op::NumpyLaplaceParam param;
+
+  NDArray** inputs = new NDArray*[2]();
+  int num_inputs = 0;
+
+  if (args[0].type_code() == kNull) {
+    param.loc = dmlc::nullopt;
+  } else if (args[0].type_code() == kNDArrayHandle) {
+    param.loc = dmlc::nullopt;
+    inputs[num_inputs] = args[0].operator mxnet::NDArray *();
+    num_inputs++;
+  } else {
+    param.loc = args[0].operator double();  // convert arg to T
+  }
+
+  if (args[1].type_code() == kNull) {
+    param.scale = dmlc::nullopt;
+  } else if (args[1].type_code() == kNDArrayHandle) {
+    param.scale = dmlc::nullopt;
+    inputs[num_inputs] = args[1].operator mxnet::NDArray *();
+    num_inputs++;
+  } else {
+    param.scale = args[1].operator double();  // convert arg to T
+  }
+
+  if (args[2].type_code() == kNull) {
+    param.size = dmlc::nullopt;
+  } else {
+    if (args[2].type_code() == kDLInt) {
+      param.size = mxnet::Tuple<int>(1, args[2].operator int64_t());
+    } else {
+      param.size = mxnet::Tuple<int>(args[2].operator ObjectRef());
+    }
+  }
+
+  if (args[3].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyLaplaceParam>(&attrs);
+  if (args[4].type_code() != kNull) {
+    attrs.dict["ctx"] = args[4].operator std::string();
+  }
+
+  inputs = inputs == nullptr ? nullptr : inputs;
+
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/random/np_laplace_op.cc b/src/operator/numpy/random/np_laplace_op.cc
new file mode 100644
index 000000000000..9d147aca7009
--- /dev/null
+++ b/src/operator/numpy/random/np_laplace_op.cc
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2019 by Contributors
+ * \file np_laplace_op.cc
+ * \brief Operator for numpy sampling from Laplace distributions
+ */
+#include "./np_laplace_op.h"
+
+namespace mxnet {
+namespace op {
+
+DMLC_REGISTER_PARAMETER(NumpyLaplaceParam);
+
+NNVM_REGISTER_OP(_npi_laplace)
+.describe("numpy behavior Laplace")
+.set_num_inputs(
+  [](const nnvm::NodeAttrs& attrs) {
+    const NumpyLaplaceParam& param = nnvm::get<NumpyLaplaceParam>(attrs.parsed);
+    int num_inputs = 2;
+    if (param.loc.has_value()) num_inputs -= 1;
+    if (param.scale.has_value()) num_inputs -= 1;
+    return num_inputs;
+  }
+)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    const NumpyLaplaceParam& param = nnvm::get<NumpyLaplaceParam>(attrs.parsed);
+    int num_inputs = 2;
+    if (param.loc.has_value()) num_inputs -= 1;
+    if (param.scale.has_value()) num_inputs -= 1;
+    if (num_inputs == 0) return std::vector<std::string>();
+    if (num_inputs == 1) return std::vector<std::string>{"input1"};
+    return std::vector<std::string>{"input1", "input2"};
+  })
+.set_attr_parser(ParamParser<NumpyLaplaceParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", TwoparamsDistOpShape<NumpyLaplaceParam>)
+.set_attr<nnvm::FInferType>("FInferType", NumpyLaplaceOpType)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const nnvm::NodeAttrs& attrs) {
+      return std::vector<ResourceRequest>{
+        ResourceRequest::kRandom, ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyLaplaceForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("input1", "NDArray-or-Symbol", "Source input")
+.add_argument("input2", "NDArray-or-Symbol", "Source input")
+.add_arguments(NumpyLaplaceParam::__FIELDS__());
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/random/np_laplace_op.cu b/src/operator/numpy/random/np_laplace_op.cu
new file mode 100644
index 000000000000..2b199e4f01d3
--- /dev/null
+++ b/src/operator/numpy/random/np_laplace_op.cu
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2019 by Contributors
+ * \file np_laplace_op.cu
+ * \brief Operator for numpy sampling from Laplace distributions
+ */
+
+#include "./np_laplace_op.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_laplace)
+.set_attr<FCompute>("FCompute<gpu>", NumpyLaplaceForward<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/random/np_laplace_op.h b/src/operator/numpy/random/np_laplace_op.h
new file mode 100644
index 000000000000..b8e829582c06
--- /dev/null
+++ b/src/operator/numpy/random/np_laplace_op.h
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2019 by Contributors
+ * \file np_laplace_op.h
+ * \brief Operator for numpy sampling from Laplace distributions
+ */
+#ifndef MXNET_OPERATOR_NUMPY_RANDOM_NP_LAPLACE_OP_H_
+#define MXNET_OPERATOR_NUMPY_RANDOM_NP_LAPLACE_OP_H_
+
+#include <mxnet/operator_util.h>
+#include <algorithm>
+#include <string>
+#include <vector>
+#include "../../elemwise_op_common.h"
+#include "../../mshadow_op.h"
+#include "../../mxnet_op.h"
+#include "../../operator_common.h"
+#include "../../tensor/elemwise_binary_broadcast_op.h"
+#include "./dist_common.h"
+
+namespace mxnet {
+namespace op {
+
+struct NumpyLaplaceParam : public dmlc::Parameter<NumpyLaplaceParam> {
+  dmlc::optional<float> loc;
+  dmlc::optional<float> scale;
+  std::string ctx;
+  int dtype;
+  dmlc::optional<mxnet::Tuple<int>> size;
+  DMLC_DECLARE_PARAMETER(NumpyLaplaceParam) {
+    DMLC_DECLARE_FIELD(loc);
+    DMLC_DECLARE_FIELD(scale);
+    DMLC_DECLARE_FIELD(size)
+      .set_default(dmlc::optional<mxnet::Tuple<int>>())
+        .describe(
+            "Output shape. If the given shape is, "
+            "e.g., (m, n, k), then m * n * k samples are drawn. "
+            "Default is None, in which case a single value is returned.");
+    DMLC_DECLARE_FIELD(ctx).set_default("cpu").describe(
+        "Context of output, in format [cpu|gpu|cpu_pinned](n)."
+        " Only used for imperative calls.");
+    DMLC_DECLARE_FIELD(dtype)
+        .add_enum("float32", mshadow::kFloat32)
+        .add_enum("float64", mshadow::kFloat64)
+        .add_enum("float16", mshadow::kFloat16)
+        .set_default(mshadow::kFloat32)
+        .describe(
+            "DType of the output in case this can't be inferred. "
+            "Defaults to float32 if not defined (dtype=None).");
+  }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream loc_s, scale_s, size_s, dtype_s;
+    loc_s << loc;
+    scale_s << scale;
+    size_s << size;
+    dtype_s << dtype;
+    (*dict)["loc"] = loc_s.str();
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["size"] = size_s.str();
+    (*dict)["dtype"] = dtype_s.str();
+    // We do not set ctx, because ctx has been set in dict instead of InitOpParam.
+    // Setting ctx here results in an error.
+  }
+};
+
+inline bool NumpyLaplaceOpType(const nnvm::NodeAttrs &attrs,
+                               std::vector<int> *in_attrs,
+                               std::vector<int> *out_attrs) {
+  const NumpyLaplaceParam &param = nnvm::get<NumpyLaplaceParam>(attrs.parsed);
+  int otype = param.dtype;
+  if (otype != -1) {
+    (*out_attrs)[0] = otype;
+  } else {
+    (*out_attrs)[0] = mshadow::kFloat32;
+  }
+  return true;
+}
+
+namespace mxnet_op {
+template <int ndim, typename IType, typename OType>
+struct laplace_kernel {
+  MSHADOW_XINLINE static void Map(index_t i, const Shape<ndim> &lstride,
+                                  const Shape<ndim> &hstride,
+                                  const Shape<ndim> &oshape, IType *loc,
+                                  IType *scale, float *uniforms, OType *out) {
+    Shape<ndim> coord = unravel(i, oshape);
+    auto lidx = static_cast<index_t>(dot(coord, lstride));
+    auto hidx = static_cast<index_t>(dot(coord, hstride));
+    IType loc_value = loc[lidx];
+    IType scale_value = scale[hidx];
+    if (uniforms[i] < 0.5) {
+      out[i] = loc_value + scale_value * log(2 * uniforms[i]);
+    } else {
+      out[i] = loc_value - scale_value * log(2 * (1 - uniforms[i]));
+    }
+  }
+};
+
+template <int ndim, typename IType, typename OType>
+struct laplace_one_scalar_kernel {
+  MSHADOW_XINLINE static void Map(index_t i, int scalar_pos,
+                                  const Shape<ndim> &stride,
+                                  const Shape<ndim> &oshape, IType *array,
+                                  float scalar, float *uniforms, OType *out) {
+    Shape<ndim> coord = unravel(i, oshape);
+    auto idx = static_cast<index_t>(dot(coord, stride));
+    IType loc_value;
+    IType scale_value;
+    if (scalar_pos == 0) {
+      loc_value = scalar;
+      scale_value = array[idx];
+    } else {
+      loc_value = array[idx];
+      scale_value = scalar;
+    }
+    if (uniforms[i] < 0.5) {
+      out[i] = loc_value + scale_value * log(2 * uniforms[i]);
+    } else {
+      out[i] = loc_value - scale_value * log(2 * (1 - uniforms[i]));
+    }
+  }
+};
+
+template <typename OType>
+struct laplace_two_scalar_kernel {
+  MSHADOW_XINLINE static void Map(index_t i, float loc, float scale,
+                                  float *uniforms, OType *out) {
+    if (uniforms[i] < 0.5) {
+      out[i] = loc + scale * log(2 * uniforms[i]);
+    } else {
+      out[i] = loc - scale * log(2 * (1 - uniforms[i]));
+    }
+  }
+};
+}  // namespace mxnet_op
+
+template <typename xpu>
+void NumpyLaplaceForward(const nnvm::NodeAttrs &attrs,
+                         const OpContext &ctx,
+                         const std::vector<TBlob> &inputs,
+                         const std::vector<OpReqType> &req,
+                         const std::vector<TBlob> &outputs) {
+  using namespace mshadow;
+  using namespace mxnet_op;
+  const NumpyLaplaceParam &param = nnvm::get<NumpyLaplaceParam>(attrs.parsed);
+  CHECK_EQ(outputs.size(), 1);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  // Generate base random number.
+  Random<xpu, float> *prnd = ctx.requested[0].get_random<xpu, float>(s);
+  Tensor<xpu, 1, float> laplace_tensor =
+      ctx.requested[1].get_space_typed<xpu, 1, float>(Shape1(outputs[0].Size()),
+                                                      s);
+  prnd->SampleUniform(&laplace_tensor, 0, 1);
+  mxnet::TShape new_lshape, new_hshape, new_oshape;
+
+  // [scalar scalar] case
+  if (inputs.size() == 0U) {
+    MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, OType, {
+      Kernel<laplace_two_scalar_kernel<OType>, xpu>::Launch(
+          s, outputs[0].Size(), param.loc.value(), param.scale.value(),
+          laplace_tensor.dptr_, outputs[0].dptr<OType>());
+    });
+  } else if (inputs.size() == 1U) {
+    // [scalar tensor], [tensor scalar] case
+    int ndim = FillShape(inputs[0].shape_, inputs[0].shape_, outputs[0].shape_,
+                         &new_lshape, &new_lshape, &new_oshape);
+    int scalar_pos;
+    float scalar_value;
+    if (param.loc.has_value()) {
+      scalar_pos = 0;
+      scalar_value = param.loc.value();
+    } else {
+      scalar_pos = 1;
+      scalar_value = param.scale.value();
+    }
+    MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, IType, {
+      MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, OType, {
+        BROADCAST_NDIM_SWITCH(ndim, NDim, {
+          Shape<NDim> oshape = new_oshape.get<NDim>();
+          Shape<NDim> stride = calc_stride(new_lshape.get<NDim>());
+          Kernel<laplace_one_scalar_kernel<NDim, IType, OType>, xpu>::Launch(
+              s, outputs[0].Size(), scalar_pos, stride, oshape,
+              inputs[0].dptr<IType>(), scalar_value, laplace_tensor.dptr_,
+              outputs[0].dptr<OType>());
+        });
+      });
+    });
+  } else if (inputs.size() == 2U) {
+    // [tensor tensor] case
+    int ndim = FillShape(inputs[0].shape_, inputs[1].shape_, outputs[0].shape_,
+                         &new_lshape, &new_hshape, &new_oshape);
+    MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, IType, {
+      MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, OType, {
+        BROADCAST_NDIM_SWITCH(ndim, NDim, {
+          Shape<NDim> oshape = new_oshape.get<NDim>();
+          Shape<NDim> lstride = calc_stride(new_lshape.get<NDim>());
+          Shape<NDim> hstride = calc_stride(new_hshape.get<NDim>());
+          Kernel<laplace_kernel<NDim, IType, OType>, xpu>::Launch(
+              s, outputs[0].Size(), lstride, hstride, oshape,
+              inputs[0].dptr<IType>(), inputs[1].dptr<IType>(),
+              laplace_tensor.dptr_, outputs[0].dptr<OType>());
+        });
+      });
+    });
+  }
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_RANDOM_NP_LAPLACE_OP_H_
diff --git a/tests/nightly/test_np_random.py b/tests/nightly/test_np_random.py
index 3d940175c46a..09ebdad90bd2 100644
--- a/tests/nightly/test_np_random.py
+++ b/tests/nightly/test_np_random.py
@@ -155,6 +155,25 @@ def generator_mx(x): return np.random.gamma(
                              nsamples=samples, nrepeat=trials)
 
 
+@retry(5)
+@with_seed()
+@use_np
+def test_np_laplace():
+    types = [None, "float32", "float64"]
+    ctx = mx.context.current_context()
+    samples = 1000000
+    # Generation test
+    trials = 8
+    num_buckets = 5
+    for dtype in types:
+        for loc, scale in [(0.0, 1.0), (1.0, 5.0)]:
+            buckets, probs = gen_buckets_probs_with_ppf(lambda x: ss.laplace.ppf(x, loc=loc, scale=scale), num_buckets)
+            buckets = np.array(buckets, dtype=dtype).tolist()
+            probs = [(buckets[i][1] - buckets[i][0])/scale for i in range(num_buckets)]
+            generator_mx_np = lambda x: np.random.laplace(loc, scale, size=x, ctx=ctx, dtype=dtype).asnumpy()
+            verify_generator(generator=generator_mx_np, buckets=buckets, probs=probs, nsamples=samples, nrepeat=trials)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 6c281cc4a125..3b29c6239e5e 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -3954,7 +3954,7 @@ def hybrid_forward(self, F, param1, param2):
 def test_np_random():
     shapes = [(), (1,), (2, 3), (4, 0, 5), 6, (7, 8), None]
     dtypes = ['float16', 'float32', 'float64']
-    op_names = ['uniform', 'normal', 'gamma']
+    op_names = ['uniform', 'normal', 'gamma', 'laplace']
     for shape in shapes:
         for dtype in dtypes:
             for op_name in op_names:

From fd171466e9b94bc4f40d20cef9a5415fb3a15569 Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Tue, 7 Apr 2020 14:39:53 +0800
Subject: [PATCH 23/44] * impl - FFi for linalg op (#17795)

* fix - cpplint

* impl - benchmark ffi for ops

* rm - FFI for ops with param

* fix - makefile

* fix - not include unordered_map and use num_inputs

* ci - compiler error

* fix - change cholesky interface

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 Makefile                                      |  4 +-
 benchmark/python/ffi/benchmark_ffi.py         |  8 ++
 python/mxnet/ndarray/numpy/linalg.py          | 18 ++---
 python/mxnet/symbol/numpy/linalg.py           |  2 +-
 src/api/operator/numpy/linalg/np_eigvals.cc   | 61 ++++++++++++++++
 src/api/operator/numpy/linalg/np_inv.cc       | 43 +++++++++++
 src/api/operator/numpy/linalg/np_pinv.cc      | 73 +++++++++++++++++++
 src/api/operator/numpy/linalg/np_potrf.cc     | 48 ++++++++++++
 src/api/operator/numpy/linalg/np_solve.cc     | 43 +++++++++++
 src/api/operator/numpy/linalg/np_tensorinv.cc | 48 ++++++++++++
 .../operator/numpy/linalg/np_tensorsolve.cc   | 56 ++++++++++++++
 src/api/operator/ufunc_helper.cc              |  1 +
 src/api/operator/utils.cc                     |  5 ++
 src/api/operator/utils.h                      |  5 +-
 src/operator/numpy/linalg/np_eigvals-inl.h    |  6 ++
 src/operator/numpy/linalg/np_pinv-inl.h       | 14 ++++
 src/operator/numpy/linalg/np_potrf.cc         |  3 +-
 src/operator/numpy/linalg/np_tensorinv-inl.h  |  6 ++
 .../numpy/linalg/np_tensorsolve-inl.h         |  6 ++
 src/operator/tensor/la_op.h                   |  6 ++
 tests/python/unittest/test_numpy_op.py        |  1 -
 21 files changed, 440 insertions(+), 17 deletions(-)
 create mode 100644 src/api/operator/numpy/linalg/np_eigvals.cc
 create mode 100644 src/api/operator/numpy/linalg/np_inv.cc
 create mode 100644 src/api/operator/numpy/linalg/np_pinv.cc
 create mode 100644 src/api/operator/numpy/linalg/np_potrf.cc
 create mode 100644 src/api/operator/numpy/linalg/np_solve.cc
 create mode 100644 src/api/operator/numpy/linalg/np_tensorinv.cc
 create mode 100644 src/api/operator/numpy/linalg/np_tensorsolve.cc

diff --git a/Makefile b/Makefile
index c050dae5e45a..e3b293230d49 100644
--- a/Makefile
+++ b/Makefile
@@ -460,9 +460,9 @@ endif
 
 all: lib/libmxnet.a lib/libmxnet.so $(BIN) extra-packages extension_libs
 
-SRC = $(wildcard src/*/*/*/*.cc src/*/*/*.cc src/*/*.cc src/*.cc)
+SRC = $(wildcard src/*/*/*/*/*.cc src/*/*/*/*.cc src/*/*/*.cc src/*/*.cc src/*.cc)
 OBJ = $(patsubst %.cc, build/%.o, $(SRC))
-CUSRC = $(wildcard src/*/*/*/*.cu src/*/*/*.cu src/*/*.cu src/*.cu)
+CUSRC = $(wildcard src/*/*/*/*.cu src/*/*/*/*.cu src/*/*/*.cu src/*/*.cu src/*.cu)
 CUOBJ = $(patsubst %.cu, build/%_gpu.o, $(CUSRC))
 
 ifeq ($(USE_TVM_OP), 1)
diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 749a322187bf..3c28b83d4561 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -61,6 +61,14 @@ def prepare_workloads():
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
+    OpArgMngr.add_workload("linalg.cholesky", pool['1x1'])
+    OpArgMngr.add_workload("linalg.eigvals", pool['1x1'])
+    OpArgMngr.add_workload("linalg.eigvalsh", pool['1x1'], UPLO='L')
+    OpArgMngr.add_workload("linalg.inv", pool['1x1'])
+    OpArgMngr.add_workload("linalg.pinv", pool['2x3x3'], pool['1'], hermitian=False)
+    OpArgMngr.add_workload("linalg.solve", pool['1x1'], pool['1'])
+    OpArgMngr.add_workload("linalg.tensorinv", pool['1x1'], ind=2)
+    OpArgMngr.add_workload("linalg.tensorsolve", pool['1x1x1'], pool['1x1x1'], (2, 0, 1))
     OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
     OpArgMngr.add_workload("argmax", pool['3x2'], axis=-1)
     OpArgMngr.add_workload("argmin", pool['3x2'], axis=-1)
diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index fdcbdac2247a..9c2344770bfa 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -92,9 +92,7 @@ def pinv(a, rcond=1e-15, hermitian=False):
     """
     if hermitian is True:
         raise NotImplementedError("hermitian is not supported yet...")
-    if _mx_nd_np._np.isscalar(rcond):
-        return _npi.pinv_scalar_rcond(a, rcond, hermitian)
-    return _npi.pinv(a, rcond, hermitian)
+    return _api_internal.pinv(a, rcond, hermitian)
 
 
 # pylint: disable=too-many-return-statements
@@ -389,7 +387,7 @@ def cholesky(a):
     array([[16.,  4.],
            [ 4., 10.]])
     """
-    return _npi.cholesky(a)
+    return _api_internal.cholesky(a, True)
 
 
 def inv(a):
@@ -431,7 +429,7 @@ def inv(a):
            [[-1.2500001 ,  0.75000006],
             [ 0.75000006, -0.25000003]]])
     """
-    return _npi.inv(a)
+    return _api_internal.inv(a)
 
 
 def det(a):
@@ -595,7 +593,7 @@ def solve(a, b):
     >>> np.allclose(np.dot(a, x), b)
     True
     """
-    return _npi.solve(a, b)
+    return _api_internal.solve(a, b)
 
 
 def tensorinv(a, ind=2):
@@ -650,7 +648,7 @@ def tensorinv(a, ind=2):
     >>> np.allclose(np.tensordot(ainv, b, 1), np.linalg.tensorsolve(a, b))
     True
     """
-    return _npi.tensorinv(a, ind)
+    return _api_internal.tensorinv(a, ind)
 
 
 def tensorsolve(a, b, axes=None):
@@ -698,7 +696,7 @@ def tensorsolve(a, b, axes=None):
     >>> np.allclose(np.tensordot(a, x, axes=3), b)
     True
     """
-    return _npi.tensorsolve(a, b, axes)
+    return _api_internal.tensorsolve(a, b, axes)
 
 
 def eigvals(a):
@@ -766,7 +764,7 @@ def eigvals(a):
     >>> LA.eigvals(A)
     array([ 1., -1.]) # random
     """
-    return _npi.eigvals(a)
+    return _api_internal.eigvals(a)
 
 
 def eigvalsh(a, UPLO='L'):
@@ -825,7 +823,7 @@ def eigvalsh(a, UPLO='L'):
     >>> LA.eigvalsh(a, UPLO='L')
     array([-2.87381886,  5.10144682,  6.38623114]) # in ascending order
     """
-    return _npi.eigvalsh(a, UPLO)
+    return _api_internal.eigvalsh(a, UPLO)
 
 
 def eig(a):
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index d326b37f0635..c05144abe4f5 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -378,7 +378,7 @@ def cholesky(a):
     array([[16.,  4.],
            [ 4., 10.]])
     """
-    return _npi.cholesky(a)
+    return _npi.cholesky(a, True)
 
 
 def inv(a):
diff --git a/src/api/operator/numpy/linalg/np_eigvals.cc b/src/api/operator/numpy/linalg/np_eigvals.cc
new file mode 100644
index 000000000000..acde49f87b74
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_eigvals.cc
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_eigvals.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_eigvals.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_eigvals-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.eigvals")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_eigvals");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+MXNET_REGISTER_API("_npi.eigvalsh")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_eigvalsh");
+  nnvm::NodeAttrs attrs;
+  op::EigvalshParam param;
+  param.UPLO = *((args[1].operator std::string()).c_str());
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::EigvalshParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_inv.cc b/src/api/operator/numpy/linalg/np_inv.cc
new file mode 100644
index 000000000000..238f666f29bd
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_inv.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_inv.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/la_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.inv")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_inv");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_pinv.cc b/src/api/operator/numpy/linalg/np_pinv.cc
new file mode 100644
index 000000000000..b14407c7b69f
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_pinv.cc
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_pinv.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_pinv.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_pinv-inl.h"
+
+namespace mxnet {
+
+inline static void _npi_pinv(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_pinv");
+  op::PinvParam param;
+  nnvm::NodeAttrs attrs;
+  param.hermitian = args[2].operator bool();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::PinvParam>(&attrs);
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+inline static void _npi_pinv_scalar_rcond(runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_pinv_scalar_rcond");
+  op::PinvScalarRcondParam param;
+  nnvm::NodeAttrs attrs;
+  param.rcond = args[1].operator double();
+  param.hermitian = args[2].operator bool();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::PinvScalarRcondParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+MXNET_REGISTER_API("_npi.pinv")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+    _npi_pinv_scalar_rcond(args, ret);
+  } else {
+    _npi_pinv(args, ret);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_potrf.cc b/src/api/operator/numpy/linalg/np_potrf.cc
new file mode 100644
index 000000000000..811ce74f8692
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_potrf.cc
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_potrf.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_potrf.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_potrf-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.cholesky")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_cholesky");
+  nnvm::NodeAttrs attrs;
+  op::LaCholeskyParam param;
+  param.lower = args[1].operator bool();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::LaCholeskyParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_solve.cc b/src/api/operator/numpy/linalg/np_solve.cc
new file mode 100644
index 000000000000..d0d263881701
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_solve.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_solve.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_solve.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.solve")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_solve");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_tensorinv.cc b/src/api/operator/numpy/linalg/np_tensorinv.cc
new file mode 100644
index 000000000000..c3062eee637f
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_tensorinv.cc
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_tensorinv.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_tensorinv.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_tensorinv-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.tensorinv")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_tensorinv");
+  nnvm::NodeAttrs attrs;
+  op::TensorinvParam param;
+  param.ind = args[1].operator int();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::TensorinvParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_tensorsolve.cc b/src/api/operator/numpy/linalg/np_tensorsolve.cc
new file mode 100644
index 000000000000..5a50c22ea94e
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_tensorsolve.cc
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_tensorsolve.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_tensorsolve.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_tensorsolve-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.tensorsolve")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_tensorsolve");
+  nnvm::NodeAttrs attrs;
+  op::TensorsolveParam param;
+  if (args[2].type_code() == kNull) {
+    param.a_axes = Tuple<int>();
+  } else {
+    if (args[2].type_code() == kDLInt) {
+      param.a_axes = Tuple<int>(1, args[2].operator int64_t());
+    } else {
+      param.a_axes = Tuple<int>(args[2].operator ObjectRef());
+    }
+  }
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::TensorsolveParam>(&attrs);
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/ufunc_helper.cc b/src/api/operator/ufunc_helper.cc
index 67bc68031417..787166df576a 100644
--- a/src/api/operator/ufunc_helper.cc
+++ b/src/api/operator/ufunc_helper.cc
@@ -23,6 +23,7 @@
  */
 #include "ufunc_helper.h"
 #include "utils.h"
+#include "../../imperative/imperative_utils.h"
 
 namespace mxnet {
 
diff --git a/src/api/operator/utils.cc b/src/api/operator/utils.cc
index 3d8401270a40..51ccb8455880 100644
--- a/src/api/operator/utils.cc
+++ b/src/api/operator/utils.cc
@@ -22,9 +22,14 @@
  * \brief Utility functions for operator invoke
  */
 #include "utils.h"
+#include "../../imperative/imperative_utils.h"
 
 namespace mxnet {
 
+bool is_recording() {
+  return Imperative::Get()->is_recording();
+}
+
 void SetInOut(std::vector<NDArray*>* ndinputs,
               std::vector<NDArray*>* ndoutputs,
               int num_inputs,
diff --git a/src/api/operator/utils.h b/src/api/operator/utils.h
index 49ee6bf2c9af..53e62ee7635b 100644
--- a/src/api/operator/utils.h
+++ b/src/api/operator/utils.h
@@ -28,7 +28,6 @@
 #include <nnvm/c_api.h>
 #include <vector>
 #include <string>
-#include "../../imperative/imperative_utils.h"
 
 namespace mxnet {
 
@@ -48,9 +47,11 @@ std::vector<NDArray*> Invoke(const nnvm::Op* op,
                              int* num_outputs,
                              NDArray** outputs);
 
+bool is_recording();
+
 template<typename T>
 void SetAttrDict(nnvm::NodeAttrs* attrs) {
-  if (Imperative::Get()->is_recording()) {
+  if (is_recording()) {
     ::dmlc::get<T>(attrs->parsed).SetAttrDict(&(attrs->dict));
   }
 }
diff --git a/src/operator/numpy/linalg/np_eigvals-inl.h b/src/operator/numpy/linalg/np_eigvals-inl.h
index 81b46d237206..26b351ac8eab 100644
--- a/src/operator/numpy/linalg/np_eigvals-inl.h
+++ b/src/operator/numpy/linalg/np_eigvals-inl.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../../operator_common.h"
 #include "../../mshadow_op.h"
 #include "../../tensor/la_op.h"
@@ -312,6 +313,11 @@ struct EigvalshParam : public dmlc::Parameter<EigvalshParam> {
     .set_default('L')
     .describe("Specifies whether the calculation is done with the lower or upper triangular part.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream UPLO_s;
+    UPLO_s << UPLO;
+    (*dict)["UPLO"] = UPLO_s.str();
+  }
 };
 
 template<typename xpu>
diff --git a/src/operator/numpy/linalg/np_pinv-inl.h b/src/operator/numpy/linalg/np_pinv-inl.h
index 76bcc9a1ab64..b3b8e0c76c64 100644
--- a/src/operator/numpy/linalg/np_pinv-inl.h
+++ b/src/operator/numpy/linalg/np_pinv-inl.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include <algorithm>
 #include "../../operator_common.h"
 #include "../../mshadow_op.h"
@@ -48,6 +49,11 @@ struct PinvParam : public dmlc::Parameter<PinvParam> {
     .set_default(false)
     .describe("If True, A is assumed to be Hermitian (symmetric if real-valued).");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream hermitian_s;
+    hermitian_s << hermitian;
+    (*dict)["hermitian"] = hermitian_s.str();
+  }
 };
 
 struct PinvScalarRcondParam : public dmlc::Parameter<PinvScalarRcondParam> {
@@ -61,6 +67,14 @@ struct PinvScalarRcondParam : public dmlc::Parameter<PinvScalarRcondParam> {
     .set_default(false)
     .describe("If True, A is assumed to be Hermitian (symmetric if real-valued).");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream rcond_s;
+    std::ostringstream hermitian_s;
+    rcond_s << rcond;
+    hermitian_s << hermitian;
+    (*dict)["rcond"] = rcond_s.str();
+    (*dict)["hermitian"] = hermitian_s.str();
+  }
 };
 
 template<typename xpu, typename DType>
diff --git a/src/operator/numpy/linalg/np_potrf.cc b/src/operator/numpy/linalg/np_potrf.cc
index cad2b3084c21..40e900872365 100644
--- a/src/operator/numpy/linalg/np_potrf.cc
+++ b/src/operator/numpy/linalg/np_potrf.cc
@@ -56,7 +56,8 @@ NNVM_REGISTER_OP(_npi_cholesky)
   { return std::vector<std::pair<int, int>>{{0, 0}}; })
 .set_attr<FCompute>("FCompute<cpu>", LaOpForward<cpu, 2, 2, 1, 1, potrf>)
 .set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseOut{"_backward_linalg_potrf"})
-.add_argument("A", "NDArray-or-Symbol", "Tensor of input matrices to be decomposed");
+.add_argument("A", "NDArray-or-Symbol", "Tensor of input matrices to be decomposed")
+.add_arguments(LaCholeskyParam::__FIELDS__());
 
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_tensorinv-inl.h b/src/operator/numpy/linalg/np_tensorinv-inl.h
index 4f92ccf9d125..414c3f09ec45 100644
--- a/src/operator/numpy/linalg/np_tensorinv-inl.h
+++ b/src/operator/numpy/linalg/np_tensorinv-inl.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../../operator_common.h"
 #include "../../mshadow_op.h"
 #include "../../tensor/la_op.h"
@@ -44,6 +45,11 @@ struct TensorinvParam : public dmlc::Parameter<TensorinvParam> {
       .set_default(2)
       .describe("Number of first indices that are involved in the inverse sum.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream ind_s;
+    ind_s << ind;
+    (*dict)["ind"] = ind_s.str();
+  }
 };
 
 template<typename xpu>
diff --git a/src/operator/numpy/linalg/np_tensorsolve-inl.h b/src/operator/numpy/linalg/np_tensorsolve-inl.h
index 829a119b64a2..bbde4d40434a 100644
--- a/src/operator/numpy/linalg/np_tensorsolve-inl.h
+++ b/src/operator/numpy/linalg/np_tensorsolve-inl.h
@@ -27,6 +27,7 @@
 
 #include <mxnet/operator_util.h>
 #include <vector>
+#include <string>
 #include "../../operator_common.h"
 #include "../../mshadow_op.h"
 #include "../../tensor/la_op.h"
@@ -46,6 +47,11 @@ struct TensorsolveParam : public dmlc::Parameter<TensorsolveParam> {
     .set_default(mxnet::Tuple<int>())
     .describe("Tuple of ints, optional. Axes in a to reorder to the right, before inversion.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_axes_s;
+    a_axes_s << a_axes;
+    (*dict)["a_axes"] = a_axes_s.str();
+  }
 };
 
 // Fix negative axes.
diff --git a/src/operator/tensor/la_op.h b/src/operator/tensor/la_op.h
index e15390ecde5a..cf80f28cb8b2 100644
--- a/src/operator/tensor/la_op.h
+++ b/src/operator/tensor/la_op.h
@@ -29,6 +29,7 @@
 #include <mxnet/imperative.h>
 #include <vector>
 #include <algorithm>
+#include <string>
 #include "../mshadow_op.h"
 #include "../mxnet_op.h"
 #include "../operator_common.h"
@@ -91,6 +92,11 @@ struct LaCholeskyParam : public dmlc::Parameter<LaCholeskyParam> {
       .describe
          ("True if the triangular matrix is lower triangular, false if it is upper triangular.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream lower_s;
+    lower_s << lower;
+    (*dict)["lower"] = lower_s.str();
+  }
 };
 
 // Parameters for matrix-matrix multiplication where one is a triangular matrix.
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 3b29c6239e5e..5bc3923aa79c 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -5976,7 +5976,6 @@ def hybrid_forward(self, F, a):
         assert mx_out.shape == np_out.shape
         assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-1, atol=1e-1)
         if grad_req != 'null':
-            print(shape, grad_req)
             mx_out.backward()
 
         # Test imperative once again

From 5a9bf61ce162d5c720c746a55d469c7c5a165f08 Mon Sep 17 00:00:00 2001
From: Haozheng Fan <fhztc1997618@gmail.com>
Date: Wed, 18 Mar 2020 05:06:01 +0800
Subject: [PATCH 24/44]  [Numpy] FFI: split and svd #17816

---
 benchmark/python/ffi/benchmark_ffi.py  |  1 +
 src/api/operator/numpy/np_matrix_op.cc | 39 ++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 3c28b83d4561..10d4984352a3 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -69,6 +69,7 @@ def prepare_workloads():
     OpArgMngr.add_workload("linalg.solve", pool['1x1'], pool['1'])
     OpArgMngr.add_workload("linalg.tensorinv", pool['1x1'], ind=2)
     OpArgMngr.add_workload("linalg.tensorsolve", pool['1x1x1'], pool['1x1x1'], (2, 0, 1))
+    OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("split", pool['3x3'], (0, 1, 2), axis=1)
     OpArgMngr.add_workload("argmax", pool['3x2'], axis=-1)
     OpArgMngr.add_workload("argmin", pool['3x2'], axis=-1)
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index fdf8e9a081fa..09fb65f0261f 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -217,4 +217,43 @@ MXNET_REGISTER_API("_npi.diag_indices_from")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.split")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_split");
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = args[2].operator int();
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+    CHECK_EQ(inputs[0]->shape()[param.axis] % param.sections, 0)
+      << "ValueError: array split does not result in an equal division";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
 }  // namespace mxnet

From 08098404495de51253a54c46df71ce65535cd4a6 Mon Sep 17 00:00:00 2001
From: Yiyan66 <57363390+Yiyan66@users.noreply.github.com>
Date: Tue, 14 Apr 2020 15:23:22 +0800
Subject: [PATCH 25/44] ffi random (#18051)

---
 benchmark/python/ffi/benchmark_ffi.py         |   7 +
 python/mxnet/ndarray/numpy/random.py          | 101 ++++--------
 python/mxnet/numpy/random.py                  |  39 +----
 python/mxnet/symbol/numpy/random.py           |  11 +-
 .../numpy/random/np_exponential_op.cc         |  71 +++++++++
 .../numpy/random/np_location_scale_op.cc      | 150 ++++++++++++++++++
 src/api/operator/numpy/random/np_pareto_op.cc |  72 +++++++++
 src/api/operator/numpy/random/np_power_op.cc  |  72 +++++++++
 .../operator/numpy/random/np_rayleigh_op.cc   |  72 +++++++++
 .../operator/numpy/random/np_weibull_op.cc    |  72 +++++++++
 src/operator/numpy/random/np_exponential_op.h |   8 +
 .../numpy/random/np_location_scale_op.h       |  10 ++
 src/operator/numpy/random/np_pareto_op.h      |   9 ++
 src/operator/numpy/random/np_power_op.h       |  13 ++
 src/operator/numpy/random/np_rayleigh_op.h    |   9 ++
 src/operator/numpy/random/np_weibull_op.h     |   9 ++
 16 files changed, 611 insertions(+), 114 deletions(-)
 create mode 100644 src/api/operator/numpy/random/np_exponential_op.cc
 create mode 100644 src/api/operator/numpy/random/np_location_scale_op.cc
 create mode 100644 src/api/operator/numpy/random/np_pareto_op.cc
 create mode 100644 src/api/operator/numpy/random/np_power_op.cc
 create mode 100644 src/api/operator/numpy/random/np_rayleigh_op.cc
 create mode 100644 src/api/operator/numpy/random/np_weibull_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 10d4984352a3..09fec7b6d7a5 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -94,6 +94,13 @@ def prepare_workloads():
     OpArgMngr.add_workload("zeros_like", pool['2x2'])
     OpArgMngr.add_workload("ones_like", pool['2x2'])
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
+    OpArgMngr.add_workload("random.exponential", scale=2, size=(2,2))
+    OpArgMngr.add_workload("random.rayleigh", scale=2, size=(2,2))
+    OpArgMngr.add_workload("random.weibull", a=2, size=(2,2))
+    OpArgMngr.add_workload("random.pareto", a=2, size=(2,2))
+    OpArgMngr.add_workload("random.power", a=2, size=(2,2))
+    OpArgMngr.add_workload("random.logistic", loc=2, scale=2, size=(2,2))
+    OpArgMngr.add_workload("random.gumbel", loc=2, scale=2, size=(2,2))
     OpArgMngr.add_workload("where", pool['2x3'], pool['2x3'], pool['2x1'])
     OpArgMngr.add_workload("may_share_memory", pool['2x3'][:0], pool['2x3'][:1])
     OpArgMngr.add_workload("diag", pool['2x2'], k=1)
diff --git a/python/mxnet/ndarray/numpy/random.py b/python/mxnet/ndarray/numpy/random.py
index 2acb1612726d..3952a6fdb561 100644
--- a/python/mxnet/ndarray/numpy/random.py
+++ b/python/mxnet/ndarray/numpy/random.py
@@ -260,24 +260,13 @@ def logistic(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
     out : ndarray or scalar
         Drawn samples from the parameterized logistic distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(loc, np_ndarray), isinstance(scale, np_ndarray))
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    if input_type == (True, True):
-        return _npi.logistic(loc, scale, loc=None, scale=None, size=size,
-                             ctx=ctx, out=out)
-    elif input_type == (False, True):
-        return _npi.logistic(scale, loc=loc, scale=None, size=size,
-                             ctx=ctx, out=out)
-    elif input_type == (True, False):
-        return _npi.logistic(loc, loc=None, scale=scale, size=size,
-                             ctx=ctx, out=out)
-    else:
-        return _npi.logistic(loc=loc, scale=scale, size=size,
-                             ctx=ctx, out=out)
+    return _api_internal.logistic(loc, scale, size, ctx, out)
 
 
 def gumbel(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
@@ -308,24 +297,13 @@ def gumbel(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
     out : ndarray or scalar
         Drawn samples from the parameterized Gumbel distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(loc, np_ndarray), isinstance(scale, np_ndarray))
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    if input_type == (True, True):
-        return _npi.gumbel(loc, scale, loc=None, scale=None, size=size,
-                           ctx=ctx, out=out)
-    elif input_type == (False, True):
-        return _npi.gumbel(scale, loc=loc, scale=None, size=size,
-                           ctx=ctx, out=out)
-    elif input_type == (True, False):
-        return _npi.gumbel(loc, loc=None, scale=scale, size=size,
-                           ctx=ctx, out=out)
-    else:
-        return _npi.gumbel(loc=loc, scale=scale, size=size,
-                           ctx=ctx, out=out)
+    return _api_internal.gumbel(loc, scale, size, ctx, out)
 
 
 def multinomial(n, pvals, size=None):
@@ -405,17 +383,13 @@ def rayleigh(scale=1.0, size=None, ctx=None, out=None):
     out : ndarray or scalar
         Drawn samples from the parameterized Rayleigh distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    tensor_type_name = np_ndarray
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    is_tensor = isinstance(scale, tensor_type_name)
-    if is_tensor:
-        return _npi.rayleigh(scale, scale=None, size=size, ctx=ctx, out=out)
-    else:
-        return _npi.rayleigh(scale=scale, size=size, ctx=ctx, out=out)
+    return _api_internal.rayleigh(scale, size, ctx, out)
 
 
 def multivariate_normal(mean, cov, size=None, check_valid=None, tol=None):
@@ -596,18 +570,13 @@ def exponential(scale=1.0, size=None, ctx=None, out=None):
     out : ndarray or scalar
         Drawn samples from the parameterized exponential distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    tensor_type_name = np_ndarray
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    is_tensor = isinstance(scale, tensor_type_name)
-    if is_tensor:
-        return _npi.exponential(scale, scale=None, size=size,
-                                ctx=ctx, out=out)
-    else:
-        return _npi.exponential(scale=scale, size=size, ctx=ctx, out=out)
+    return _api_internal.exponential(scale, size, ctx, out)
 
 
 def weibull(a, size=None, ctx=None, out=None):
@@ -652,17 +621,13 @@ def weibull(a, size=None, ctx=None, out=None):
     model time to failure, in modeling particle sizes, in information retrieval
     to model dwell time on pages, in quantitative finance to model risk etc.
     """
-    from ...numpy import ndarray as np_ndarray
-    tensor_type_name = np_ndarray
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    is_tensor = isinstance(a, tensor_type_name)
-    if is_tensor:
-        return _npi.weibull(a, a=None, size=size, ctx=ctx, out=out)
-    else:
-        return _npi.weibull(a=a, size=size, ctx=ctx, out=out)
+    return _api_internal.weibull(a, size, ctx, out)
 
 
 def pareto(a, size=None, ctx=None, out=None):
@@ -697,20 +662,16 @@ def pareto(a, size=None, ctx=None, out=None):
     where a is the shape and m the scale. Here m is assumed 1. The Pareto distribution
     is a power law distribution. Pareto created it to describe the wealth in the economy.
     """
-    from ...numpy import ndarray as np_ndarray
-    tensor_type_name = np_ndarray
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    is_tensor = isinstance(a, tensor_type_name)
-    if is_tensor:
-        return _npi.pareto(a, a=None, size=size, ctx=ctx, out=out)
-    else:
-        return _npi.pareto(a=a, size=size, ctx=ctx, out=out)
+    return _api_internal.pareto(a, size, ctx, out)
 
 
-def power(a, size=None):
+def power(a, size=None, ctx=None, out=None):
     r"""Draw samples in [0, 1] from a power distribution with given parameter a.
 
     Parameters
@@ -742,15 +703,13 @@ def power(a, size=None):
     The power distribution is just the inverse of the Pareto distribution and
     a special case of the Beta distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    tensor_type_name = np_ndarray
+    if ctx is None:
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
-    is_tensor = isinstance(a, tensor_type_name)
-    if is_tensor:
-        return _npi.powerd(a, a=None, size=size)
-    else:
-        return _npi.powerd(a=a, size=size)
+    return _api_internal.powerd(a, size, ctx, out)
 
 
 def gamma(shape, scale=1.0, size=None, dtype=None, ctx=None, out=None):
diff --git a/python/mxnet/numpy/random.py b/python/mxnet/numpy/random.py
index c6690f149fe8..6d46b2d314aa 100644
--- a/python/mxnet/numpy/random.py
+++ b/python/mxnet/numpy/random.py
@@ -270,10 +270,8 @@ def lognormal(mean=0.0, sigma=1.0, size=None, dtype=None, ctx=None, out=None):
 
 def logistic(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
     r"""Draw samples from a logistic distribution.
-
     Samples are drawn from a logistic distribution with specified
     parameters, loc (location or mean, also median), and scale (>0).
-
     Parameters
     ----------
     loc : float or array_like of floats, optional
@@ -290,23 +288,18 @@ def logistic(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
         Device context of output, default is current context.
     out : ``ndarray``, optional
         Store output to an existing ``ndarray``.
-
     Returns
     -------
     out : ndarray or scalar
         Drawn samples from the parameterized logistic distribution.
-
     Examples
     --------
     Draw samples from the distribution:
-
     >>> loc, scale = 10, 1
     >>> s = np.random.logistic(loc, scale, 10000)
     >>> import matplotlib.pyplot as plt
     >>> count, bins, ignored = plt.hist(s, bins=50)
-
     #   plot against distribution
-
     >>> def logist(x, loc, scale):
     ...     return np.exp((loc-x)/scale)/(scale*(1+np.exp((loc-x)/scale))**2)
     >>> lgst_val = logist(bins, loc, scale)
@@ -318,10 +311,8 @@ def logistic(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
 
 def gumbel(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
     r"""Draw samples from a Gumbel distribution.
-
     Draw samples from a Gumbel distribution with specified location and
     scale.
-
     Parameters
     ----------
     loc : float or array_like of floats, optional
@@ -338,32 +329,25 @@ def gumbel(loc=0.0, scale=1.0, size=None, ctx=None, out=None):
         Device context of output, default is current context.
     out : ``ndarray``, optional
         Store output to an existing ``ndarray``.
-
     Returns
     -------
     out : ndarray or scalar
         Drawn samples from the parameterized Gumbel distribution.
-
     Examples
     --------
     Draw samples from the distribution:
-
     >>> mu, beta = 0, 0.1 # location and scale
     >>> s = np.random.gumbel(mu, beta, 1000)
-
     Display the histogram of the samples, along with
     the probability density function:
-
     >>> import matplotlib.pyplot as plt
     >>> count, bins, ignored = plt.hist(s, 30, density=True)
     >>> plt.plot(bins, (1/beta)*np.exp(-(bins - mu)/beta)
     ...          * np.exp( -np.exp( -(bins - mu) /beta) ),
     ...          linewidth=2, color='r')
     >>> plt.show()
-
     Show how an extreme value distribution can arise from a Gaussian process
     and compare to a Gaussian:
-
     >>> means = []
     >>> maxima = []
     >>> for i in range(0,1000) :
@@ -561,10 +545,8 @@ def choice(a, size=None, replace=True, p=None, ctx=None, out=None):
 
 def rayleigh(scale=1.0, size=None, ctx=None, out=None):
     r"""Draw samples from a Rayleigh distribution.
-
     The :math:`\chi` and Weibull distributions are generalizations of the
     Rayleigh.
-
     Parameters
     ----------
     scale : float, optional
@@ -578,7 +560,6 @@ def rayleigh(scale=1.0, size=None, ctx=None, out=None):
         Device context of output, default is current context.
     out : ``ndarray``, optional
         Store output to an existing ``ndarray``.
-
     Returns
     -------
     out : ndarray or scalar
@@ -616,7 +597,6 @@ def rand(*size, **kwargs):
 
 def exponential(scale=1.0, size=None, ctx=None, out=None):
     r"""Draw samples from an exponential distribution.
-
     Parameters
     ----------
     scale : float or array_like of floats
@@ -631,7 +611,6 @@ def exponential(scale=1.0, size=None, ctx=None, out=None):
         Device context of output, default is current context.
     out : ``ndarray``, optional
         Store output to an existing ``ndarray``.
-
     Returns
     -------
     out : ndarray or scalar
@@ -643,7 +622,6 @@ def exponential(scale=1.0, size=None, ctx=None, out=None):
 def weibull(a, size=None, ctx=None, out=None):
     r"""Draw samples from a 1-parameter Weibull distribution with given parameter a
     via inversion.
-
     Parameters
     ----------
     a : float or array_like of floats
@@ -661,23 +639,18 @@ def weibull(a, size=None, ctx=None, out=None):
     --------
     >>> np.random.weibull(a=5)
     array(0.9553641)
-
     >>> np.random.weibull(a=5, size=[2,3])
     array([[1.0466299 , 1.1320982 , 0.98415005],
           [1.1430776 , 0.9532727 , 1.1344457 ]])
-
     >>> np.random.weibull(a=np.array([2,3])
     array([0.98843634, 1.0125613 ])
-
     The Weibull distribution is one of a class of Generalized Extreme
     Value (GEV) distributions. This class includes the Gumbel and Frechet
     distributions.
-
     The probability density for the Weibull distribution is
     f(x) = \frac{a}{\lambda}(\frac{x}{\lambda})^{a-1}e^{-(x/\lambda)^a},
     where a is the shape and \lambda the scale. The generated 1-parameter Weibull
     sample has the scale parameter \lambda = 1.
-
     The Weibull distribution is commonly used in reliability engineering to
     model time to failure, in modeling particle sizes, in information retrieval
     to model dwell time on pages, in quantitative finance to model risk etc.
@@ -687,7 +660,6 @@ def weibull(a, size=None, ctx=None, out=None):
 
 def pareto(a, size=None, ctx=None, out=None):
     r"""Draw samples from a Pareto II or Lomax distribution with specified shape a.
-
     Parameters
     ----------
     a : float or array_like of floats
@@ -697,12 +669,10 @@ def pareto(a, size=None, ctx=None, out=None):
         ``m * n * k`` samples are drawn.  If size is ``None`` (default),
         a single value is returned if ``a`` is a scalar. Otherwise,
         ``np.array(a).size`` samples are drawn.
-
     Returns
     -------
     out : ndarray or scalar
         Drawn samples from the Pareto distribution.
-
     Examples
     --------
     >>> np.random.pareto(a=5)
@@ -712,7 +682,6 @@ def pareto(a, size=None, ctx=None, out=None):
             [0.0311172 , 0.12911797, 0.03370714]])
     >>> np.random.pareto(a=np.array([2,3])
     array([0.26636696, 0.15685666])
-
     The probability density for the Pareto distribution is f(x) = \frac{am^a}{x^{a+1}}
     where a is the shape and m the scale. Here m is assumed 1. The Pareto distribution
     is a power law distribution. Pareto created it to describe the wealth in the economy.
@@ -720,9 +689,8 @@ def pareto(a, size=None, ctx=None, out=None):
     return _mx_nd_np.random.pareto(a, size=size, ctx=ctx, out=out)
 
 
-def power(a, size=None):
+def power(a, size=None, ctx=None, out=None):
     r"""Draw samples in [0, 1] from a power distribution with given parameter a.
-
     Parameters
     ----------
     a : float or array_like of floats
@@ -732,12 +700,10 @@ def power(a, size=None):
         ``m * n * k`` samples are drawn.  If size is ``None`` (default),
         a single value is returned if ``a`` is a scalar. Otherwise,
         ``np.array(a).size`` samples are drawn.
-
     Returns
     -------
     out : ndarray or scalar
         Drawn samples from the power distribution.
-
     Examples
     --------
     >>> np.random.power(a=5)
@@ -747,12 +713,11 @@ def power(a, size=None):
            [0.9078098 , 0.87819266, 0.730635]])
     >>> np.random.power(a=np.array([2,3])
     array([0.7499419 , 0.88894516])
-
     The probability density function is f(x; a) = ax^{a-1}, 0 \le x \le 1, a>0.
     The power distribution is just the inverse of the Pareto distribution and
     a special case of the Beta distribution.
     """
-    return _mx_nd_np.random.power(a, size)
+    return _mx_nd_np.random.power(a, size=size, ctx=ctx, out=out)
 
 
 def shuffle(x):
diff --git a/python/mxnet/symbol/numpy/random.py b/python/mxnet/symbol/numpy/random.py
index e6e729f19fe4..41664c526366 100644
--- a/python/mxnet/symbol/numpy/random.py
+++ b/python/mxnet/symbol/numpy/random.py
@@ -527,10 +527,8 @@ def gamma(shape, scale=1.0, size=None, dtype=None, ctx=None, out=None):
 
 def rayleigh(scale=0.0, size=None, ctx=None, out=None):
     r"""Draw samples from a Rayleigh distribution.
-
     The :math:`\chi` and Weibull distributions are generalizations of the
     Rayleigh.
-
     Parameters
     ----------
     scale : float or _Symbol
@@ -542,7 +540,6 @@ def rayleigh(scale=0.0, size=None, ctx=None, out=None):
         ``np.array(scale).size`` samples are drawn.
     ctx : Context, optional
         Device context of output. Default is current context.
-
     Returns
     -------
     out : _Symbol
@@ -863,7 +860,7 @@ def pareto(a, size=None, ctx=None, out=None):
         return _npi.pareto(a=a, size=size, ctx=ctx, out=out)
 
 
-def power(a, size=None):
+def power(a, size=None, ctx=None, out=None):
     r"""Draw samples in [0, 1] from a power distribution with given parameter a.
 
     Parameters
@@ -897,13 +894,15 @@ def power(a, size=None):
     """
     from ..numpy import _Symbol as np_symbol
     tensor_type_name = np_symbol
+    if ctx is None:
+        ctx = current_context()
     if size == ():
         size = None
     is_tensor = isinstance(a, tensor_type_name)
     if is_tensor:
-        return _npi.powerd(a, a=None, size=size)
+        return _npi.powerd(a, a=None, size=size, ctx=ctx, out=out)
     else:
-        return _npi.powerd(a=a, size=size)
+        return _npi.powerd(a=a, size=size, ctx=ctx, out=out)
 
 
 def multivariate_normal(mean, cov, size=None, check_valid=None, tol=None):
diff --git a/src/api/operator/numpy/random/np_exponential_op.cc b/src/api/operator/numpy/random/np_exponential_op.cc
new file mode 100644
index 000000000000..fbb1644c6c5a
--- /dev/null
+++ b/src/api/operator/numpy/random/np_exponential_op.cc
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_exponential_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_exponential_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_exponential_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.exponential")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_exponential");
+  op::NumpyExponentialParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[1].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[1].operator int64_t());
+  } else if (args[1].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[1];
+  int num_inputs = 0;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    param.scale = args[0].operator double();
+    num_inputs = 0;
+  } else {
+    param.scale = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyExponentialParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_location_scale_op.cc b/src/api/operator/numpy/random/np_location_scale_op.cc
new file mode 100644
index 000000000000..d4702fc96404
--- /dev/null
+++ b/src/api/operator/numpy/random/np_location_scale_op.cc
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_location_scale_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_location_scale_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_location_scale_op.h"
+
+namespace mxnet {
+
+int scalar_number(const runtime::MXNetArgs& args) {
+    int result = 0;
+    if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt)
+         result++;
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt)
+        result++;
+    return result;
+}
+
+MXNET_REGISTER_API("_npi.gumbel")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_gumbel");
+  op::NumpyLocationScaleParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[2].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[2].operator int64_t());
+  } else if (args[2].type_code() == kNull) {
+      param.size = Tuple<int>({1});
+  } else {
+      param.size = Tuple<int>(args[2].operator ObjectRef());
+  }
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  NDArray* out = args[4].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  int scalar = scalar_number(args);
+  NDArray* inputs[2];
+  int num_inputs = 0;
+  if (scalar == 2) {
+    param.loc = args[0].operator double();
+    param.scale = args[1].operator double();
+  } else if (scalar == 0) {
+    param.loc = dmlc::nullopt;
+    param.scale = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    inputs[1] = args[1].operator mxnet::NDArray*();
+    num_inputs = 2;
+  } else {
+    if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+      param.loc = dmlc::nullopt;
+      param.scale = args[1].operator double();
+      inputs[0] = args[0].operator mxnet::NDArray*();
+    } else {
+      param.loc = args[0].operator double();
+      param.scale = dmlc::nullopt;
+      inputs[0] = args[1].operator mxnet::NDArray*();
+    }
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyLocationScaleParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(4);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+MXNET_REGISTER_API("_npi.logistic")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_logistic");
+  op::NumpyLocationScaleParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[2].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[2].operator int64_t());
+  } else if (args[2].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[2].operator ObjectRef());
+  }
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  NDArray* out = args[4].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  int scalar = scalar_number(args);
+  NDArray* inputs[2];
+  int num_inputs = 0;
+  if (scalar == 2) {
+    param.loc = args[0].operator double();
+    param.scale = args[1].operator double();
+  } else if (scalar == 0) {
+    param.loc = dmlc::nullopt;
+    param.scale = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    inputs[1] = args[1].operator mxnet::NDArray*();
+    num_inputs = 2;
+  } else {
+    if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+      param.loc = dmlc::nullopt;
+      param.scale = args[1].operator double();
+      inputs[0] = args[0].operator mxnet::NDArray*();
+    } else {
+      param.loc = args[0].operator double();
+      param.scale = dmlc::nullopt;
+      inputs[0] = args[1].operator mxnet::NDArray*();
+    }
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyLocationScaleParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(4);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_pareto_op.cc b/src/api/operator/numpy/random/np_pareto_op.cc
new file mode 100644
index 000000000000..92e3645b75bd
--- /dev/null
+++ b/src/api/operator/numpy/random/np_pareto_op.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_pareto_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_pareto_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_pareto_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.pareto")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_pareto");
+  op::NumpyParetoParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[1].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[1].operator int64_t());
+  } else if (args[1].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[1];
+  int num_inputs = 0;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    param.a = args[0].operator double();
+    num_inputs = 0;
+  } else {
+    param.a = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyParetoParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_power_op.cc b/src/api/operator/numpy/random/np_power_op.cc
new file mode 100644
index 000000000000..12a621726cd2
--- /dev/null
+++ b/src/api/operator/numpy/random/np_power_op.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_power_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_power_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_power_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.powerd")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_powerd");
+  op::NumpyPowerParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[1].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[1].operator int64_t());
+  } else if (args[1].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[1];
+  int num_inputs = 0;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    param.a = args[0].operator double();
+    num_inputs = 0;
+  } else {
+    param.a = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyPowerParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_rayleigh_op.cc b/src/api/operator/numpy/random/np_rayleigh_op.cc
new file mode 100644
index 000000000000..428e433763ad
--- /dev/null
+++ b/src/api/operator/numpy/random/np_rayleigh_op.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_rayleigh_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_rayleigh_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_rayleigh_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.rayleigh")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_rayleigh");
+  op::NumpyRayleighParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[1].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[1].operator int64_t());
+  } else if (args[1].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[1];
+  int num_inputs = 0;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    param.scale = args[0].operator double();
+    num_inputs = 0;
+  } else {
+    param.scale = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyRayleighParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_weibull_op.cc b/src/api/operator/numpy/random/np_weibull_op.cc
new file mode 100644
index 000000000000..ef3b7e6ed7b6
--- /dev/null
+++ b/src/api/operator/numpy/random/np_weibull_op.cc
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_weibull_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_weibull_op.h
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_weibull_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.weibull")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_weibull");
+  op::NumpyWeibullParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  if (args[1].type_code() == kDLInt) {
+      param.size = Tuple<int>(1, args[1].operator int64_t());
+  } else if (args[1].type_code() == kNull) {
+      param.size = dmlc::nullopt;
+  } else {
+      param.size = Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  NDArray* inputs[1];
+  int num_inputs = 0;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    param.a = args[0].operator double();
+    num_inputs = 0;
+  } else {
+    param.a = dmlc::nullopt;
+    inputs[0] = args[0].operator mxnet::NDArray*();
+    num_inputs = 1;
+  }
+  attrs.parsed = std::move(param);
+  SetAttrDict<op::NumpyWeibullParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs,
+                          &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/random/np_exponential_op.h b/src/operator/numpy/random/np_exponential_op.h
index 25593063872d..36d29ff842e3 100644
--- a/src/operator/numpy/random/np_exponential_op.h
+++ b/src/operator/numpy/random/np_exponential_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -57,6 +58,13 @@ struct NumpyExponentialParam : public dmlc::Parameter<NumpyExponentialParam> {
         "Context of output, in format [cpu|gpu|cpu_pinned](n)."
         " Only used for imperative calls.");
   }
+    void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream scale_s, size_s;
+    scale_s << scale;
+    size_s << size;
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 template <typename DType>
diff --git a/src/operator/numpy/random/np_location_scale_op.h b/src/operator/numpy/random/np_location_scale_op.h
index 558bdcd5b267..00c89c149c5c 100644
--- a/src/operator/numpy/random/np_location_scale_op.h
+++ b/src/operator/numpy/random/np_location_scale_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -59,6 +60,15 @@ struct NumpyLocationScaleParam : public dmlc::Parameter<NumpyLocationScaleParam>
         "Context of output, in format [cpu|gpu|cpu_pinned](n)."
         " Only used for imperative calls.");
   }
+    void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream loc_s, scale_s, size_s;
+    loc_s << loc;
+    scale_s << scale;
+    size_s << size;
+    (*dict)["loc"] = loc_s.str();
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 inline bool NumpyLocationScaleOpType(const nnvm::NodeAttrs &attrs,
diff --git a/src/operator/numpy/random/np_pareto_op.h b/src/operator/numpy/random/np_pareto_op.h
index 85eab97aef8c..a8a5d7f411c0 100644
--- a/src/operator/numpy/random/np_pareto_op.h
+++ b/src/operator/numpy/random/np_pareto_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -57,6 +58,14 @@ struct NumpyParetoParam : public dmlc::Parameter<NumpyParetoParam> {
         "Context of output, in format [cpu|gpu|cpu_pinned](n)."
         " Only used for imperative calls.");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_s, size_s;
+    a_s << a;
+    size_s << size;
+    (*dict)["a"] = a_s.str();
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 template <typename DType>
diff --git a/src/operator/numpy/random/np_power_op.h b/src/operator/numpy/random/np_power_op.h
index a8835fd62957..dae730285902 100644
--- a/src/operator/numpy/random/np_power_op.h
+++ b/src/operator/numpy/random/np_power_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -43,6 +44,7 @@ namespace op {
 
 struct NumpyPowerParam : public dmlc::Parameter<NumpyPowerParam> {
   dmlc::optional<float> a;
+  std::string ctx;
   dmlc::optional<mxnet::Tuple<int>> size;
   DMLC_DECLARE_PARAMETER(NumpyPowerParam) {
       DMLC_DECLARE_FIELD(a)
@@ -52,6 +54,17 @@ struct NumpyPowerParam : public dmlc::Parameter<NumpyPowerParam> {
       .describe("Output shape. If the given shape is, "
           "e.g., (m, n, k), then m * n * k samples are drawn. "
           "Default is None, in which case a single value is returned.");
+      DMLC_DECLARE_FIELD(ctx).set_default("cpu").describe(
+        "Context of output, in format [cpu|gpu|cpu_pinned](n)."
+        " Only used for imperative calls.");
+  }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_s, size_s;
+    a_s << a;
+    size_s << size;
+    (*dict)["a"] = a_s.str();
+    (*dict)["size"] = size_s.str();
   }
 };
 
diff --git a/src/operator/numpy/random/np_rayleigh_op.h b/src/operator/numpy/random/np_rayleigh_op.h
index 9b222684188b..3444f3b74af5 100644
--- a/src/operator/numpy/random/np_rayleigh_op.h
+++ b/src/operator/numpy/random/np_rayleigh_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -57,6 +58,14 @@ struct NumpyRayleighParam : public dmlc::Parameter<NumpyRayleighParam> {
         "Context of output, in format [cpu|gpu|cpu_pinned](n)."
         " Only used for imperative calls.");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream scale_s, size_s;
+    scale_s << scale;
+    size_s << size;
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 template <typename DType>
diff --git a/src/operator/numpy/random/np_weibull_op.h b/src/operator/numpy/random/np_weibull_op.h
index afb37288b04e..ff4c40ae8db5 100644
--- a/src/operator/numpy/random/np_weibull_op.h
+++ b/src/operator/numpy/random/np_weibull_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <vector>
 #include <cmath>
+#include <unordered_map>
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -57,6 +58,14 @@ struct NumpyWeibullParam : public dmlc::Parameter<NumpyWeibullParam> {
         "Context of output, in format [cpu|gpu|cpu_pinned](n)."
         " Only used for imperative calls.");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_s, size_s;
+    a_s << a;
+    size_s << size;
+    (*dict)["a"] = a_s.str();
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 template <typename DType>

From 9a3a4e6dd334010d13c1a1995fb0d3483a33a0df Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Wed, 8 Apr 2020 08:00:23 +0800
Subject: [PATCH 26/44] * impl - linalg.lstsq for cpu (#17950)

* impl - linalg.lstsq for gpu

* fix - use NDArray to solve problem that outputs shape can not be derived

* fix - sanity

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 python/mxnet/ndarray/numpy/linalg.py          |  81 ++-
 python/mxnet/numpy/fallback_linalg.py         |   2 -
 python/mxnet/numpy/linalg.py                  |  73 ++-
 python/mxnet/numpy_dispatch_protocol.py       |   1 +
 python/mxnet/symbol/numpy/linalg.py           |  67 +-
 src/operator/c_lapack_api.cc                  |  12 +
 src/operator/c_lapack_api.h                   |  64 +-
 src/operator/numpy/linalg/np_lstsq-inl.h      | 593 ++++++++++++++++++
 src/operator/numpy/linalg/np_lstsq.cc         |  97 +++
 src/operator/numpy/linalg/np_lstsq.cu         |  35 ++
 .../unittest/test_numpy_interoperability.py   |  41 +-
 tests/python/unittest/test_numpy_op.py        |  76 +++
 12 files changed, 1124 insertions(+), 18 deletions(-)
 create mode 100644 src/operator/numpy/linalg/np_lstsq-inl.h
 create mode 100644 src/operator/numpy/linalg/np_lstsq.cc
 create mode 100644 src/operator/numpy/linalg/np_lstsq.cu

diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index 9c2344770bfa..e153ccb792a5 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -17,12 +17,91 @@
 
 """Namespace for operators used in Gluon dispatched by F=ndarray."""
 
+import numpy as _np
 from . import _op as _mx_nd_np
 from . import _internal as _npi
 from . import _api_internal
 
 __all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh']
+           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+
+
+def lstsq(a, b, rcond='warn'):
+    r"""
+    Return the least-squares solution to a linear matrix equation.
+
+    Solves the equation :math:`a x = b` by computing a vector `x` that
+    minimizes the squared Euclidean 2-norm :math:`\| b - a x \|^2_2`.
+    The equation may be under-, well-, or over-determined (i.e., the
+    number of linearly independent rows of `a` can be less than, equal
+    to, or greater than its number of linearly independent columns).
+    If `a` is square and of full rank, then `x` (but for round-off error)
+    is the "exact" solution of the equation.
+
+    Parameters
+    ----------
+    a : (M, N) ndarray
+        "Coefficient" matrix.
+    b : {(M,), (M, K)} ndarray
+        Ordinate or "dependent variable" values. If `b` is two-dimensional,
+        the least-squares solution is calculated for each of the `K` columns
+        of `b`.
+    rcond : float, optional
+        Cut-off ratio for small singular values of `a`.
+        For the purposes of rank determination, singular values are treated
+        as zero if they are smaller than `rcond` times the largest singular
+        value of `a`
+        The default of ``warn`` or ``-1`` will use the machine precision as
+        `rcond` parameter. The default of ``None`` will use the machine
+        precision times `max(M, N)`.
+
+    Returns
+    -------
+    x : {(N,), (N, K)} ndarray
+        Least-squares solution. If `b` is two-dimensional,
+        the solutions are in the `K` columns of `x`.
+    residuals : {(1,), (K,), (0,)} ndarray
+        Sums of residuals.
+        Squared Euclidean 2-norm for each column in ``b - a*x``.
+        If the rank of `a` is < N or M <= N, this is an empty array.
+        If `b` is 1-dimensional, this is a (1,) shape array.
+        Otherwise the shape is (K,).
+    rank : int
+        Rank of matrix `a`.
+    s : (min(M, N),) ndarray
+        Singular values of `a`.
+
+    Raises
+    ------
+    MXNetError
+        If computation does not converge.
+
+    Notes
+    -----
+    If `b` is a matrix, then all array results are returned as matrices.
+
+    Examples
+    --------
+    >>> x = np.array([0, 1, 2, 3])
+    >>> y = np.array([-1, 0.2, 0.9, 2.1])
+    >>> A = np.vstack([x, np.ones(len(x))]).T
+    >>> A
+    array([[ 0.,  1.],
+           [ 1.,  1.],
+           [ 2.,  1.],
+           [ 3.,  1.]])
+    >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0]
+    >>> m, c
+    (1.0 -0.95) # may vary
+    """
+    new_default = False
+    if rcond is None:
+        rcond = _np.finfo(a.dtype).eps
+        new_default = True
+    if rcond == "warn":
+        rcond = -1
+    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, new_default=new_default)
+    return (x, residuals, rank, s)
 
 
 def pinv(a, rcond=1e-15, hermitian=False):
diff --git a/python/mxnet/numpy/fallback_linalg.py b/python/mxnet/numpy/fallback_linalg.py
index 12e0e10779cc..944a298453ab 100644
--- a/python/mxnet/numpy/fallback_linalg.py
+++ b/python/mxnet/numpy/fallback_linalg.py
@@ -23,7 +23,6 @@
 
 __all__ = [
     'cond',
-    'lstsq',
     'matrix_power',
     'matrix_rank',
     'multi_dot',
@@ -31,7 +30,6 @@
 ]
 
 cond = onp.linalg.cond
-lstsq = onp.linalg.lstsq
 matrix_power = onp.linalg.matrix_power
 matrix_rank = onp.linalg.matrix_rank
 multi_dot = onp.linalg.multi_dot
diff --git a/python/mxnet/numpy/linalg.py b/python/mxnet/numpy/linalg.py
index 4d0c5b01698e..f212c41a7872 100644
--- a/python/mxnet/numpy/linalg.py
+++ b/python/mxnet/numpy/linalg.py
@@ -22,10 +22,81 @@
 from . import fallback_linalg
 
 __all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh']
+           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
 __all__ += fallback_linalg.__all__
 
 
+def lstsq(a, b, rcond='warn'):
+    r"""
+    Return the least-squares solution to a linear matrix equation.
+
+    Solves the equation :math:`a x = b` by computing a vector `x` that
+    minimizes the squared Euclidean 2-norm :math:`\| b - a x \|^2_2`.
+    The equation may be under-, well-, or over-determined (i.e., the
+    number of linearly independent rows of `a` can be less than, equal
+    to, or greater than its number of linearly independent columns).
+    If `a` is square and of full rank, then `x` (but for round-off error)
+    is the "exact" solution of the equation.
+
+    Parameters
+    ----------
+    a : (M, N) ndarray
+        "Coefficient" matrix.
+    b : {(M,), (M, K)} ndarray
+        Ordinate or "dependent variable" values. If `b` is two-dimensional,
+        the least-squares solution is calculated for each of the `K` columns
+        of `b`.
+    rcond : float, optional
+        Cut-off ratio for small singular values of `a`.
+        For the purposes of rank determination, singular values are treated
+        as zero if they are smaller than `rcond` times the largest singular
+        value of `a`
+        The default of ``warn`` or ``-1`` will use the machine precision as
+        `rcond` parameter. The default of ``None`` will use the machine
+        precision times `max(M, N)`.
+
+    Returns
+    -------
+    x : {(N,), (N, K)} ndarray
+        Least-squares solution. If `b` is two-dimensional,
+        the solutions are in the `K` columns of `x`.
+    residuals : {(1,), (K,), (0,)} ndarray
+        Sums of residuals.
+        Squared Euclidean 2-norm for each column in ``b - a*x``.
+        If the rank of `a` is < N or M <= N, this is an empty array.
+        If `b` is 1-dimensional, this is a (1,) shape array.
+        Otherwise the shape is (K,).
+    rank : int
+        Rank of matrix `a`.
+    s : (min(M, N),) ndarray
+        Singular values of `a`.
+
+    Raises
+    ------
+    MXNetError
+        If computation does not converge.
+
+    Notes
+    -----
+    If `b` is a matrix, then all array results are returned as matrices.
+
+    Examples
+    --------
+    >>> x = np.array([0, 1, 2, 3])
+    >>> y = np.array([-1, 0.2, 0.9, 2.1])
+    >>> A = np.vstack([x, np.ones(len(x))]).T
+    >>> A
+    array([[ 0.,  1.],
+           [ 1.,  1.],
+           [ 2.,  1.],
+           [ 3.,  1.]])
+    >>> m, c = np.linalg.lstsq(A, y, rcond=None)[0]
+    >>> m, c
+    (1.0 -0.95) # may vary
+    """
+    return _mx_nd_np.linalg.lstsq(a, b, rcond)
+
+
 def pinv(a, rcond=1e-15, hermitian=False):
     r"""
     Compute the (Moore-Penrose) pseudo-inverse of a matrix.
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index 781ec55b3796..6357fe779dbc 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -158,6 +158,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'linalg.solve',
     'linalg.tensorinv',
     'linalg.tensorsolve',
+    'linalg.lstsq',
     'linalg.pinv',
     'linalg.eigvals',
     'linalg.eig',
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index c05144abe4f5..c34cceaffb74 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -17,12 +17,77 @@
 
 """Namespace for operators used in Gluon dispatched by F=symbol."""
 
+import numpy as _np
 from . import _symbol
 from . import _op as _mx_sym_np
 from . import _internal as _npi
 
 __all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh']
+           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+
+
+def lstsq(a, b, rcond='warn'):
+    r"""
+    Return the least-squares solution to a linear matrix equation.
+
+    Solves the equation :math:`a x = b` by computing a vector `x` that
+    minimizes the squared Euclidean 2-norm :math:`\| b - a x \|^2_2`.
+    The equation may be under-, well-, or over-determined (i.e., the
+    number of linearly independent rows of `a` can be less than, equal
+    to, or greater than its number of linearly independent columns).
+    If `a` is square and of full rank, then `x` (but for round-off error)
+    is the "exact" solution of the equation.
+
+    Parameters
+    ----------
+    a : (M, N) _Symbol
+        "Coefficient" matrix.
+    b : {(M,), (M, K)} _Symbol
+        Ordinate or "dependent variable" values. If `b` is two-dimensional,
+        the least-squares solution is calculated for each of the `K` columns
+        of `b`.
+    rcond : float, optional
+        Cut-off ratio for small singular values of `a`.
+        For the purposes of rank determination, singular values are treated
+        as zero if they are smaller than `rcond` times the largest singular
+        value of `a`
+        The default of ``warn`` or ``-1`` will use the machine precision as
+        `rcond` parameter. The default of ``None`` will use the machine
+        precision times `max(M, N)` as `rcond` parameter.
+
+    Returns
+    -------
+    x : {(N,), (N, K)} _Symbol
+        Least-squares solution. If `b` is two-dimensional,
+        the solutions are in the `K` columns of `x`.
+    residuals : {(1,), (K,), (0,)} _Symbol
+        Sums of residuals.
+        Squared Euclidean 2-norm for each column in ``b - a*x``.
+        If the rank of `a` is < N or M <= N, this is an empty array.
+        If `b` is 1-dimensional, this is a (1,) shape array.
+        Otherwise the shape is (K,).
+    rank : int
+        Rank of matrix `a`.
+    s : (min(M, N),) _Symbol
+        Singular values of `a`.
+
+    Raises
+    ------
+    MXNetError
+        If computation does not converge.
+
+    Notes
+    -----
+    If `b` is a matrix, then all array results are returned as matrices.
+    """
+    new_default = False
+    if rcond is None:
+        rcond = _np.finfo(_np.float64).eps
+        new_default = True
+    if rcond == "warn":
+        rcond = -1
+    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, new_default=new_default)
+    return (x, residuals, rank, s)
 
 
 def pinv(a, rcond=1e-15, hermitian=False):
diff --git a/src/operator/c_lapack_api.cc b/src/operator/c_lapack_api.cc
index 57e83b1ddb2d..90ce94812a59 100644
--- a/src/operator/c_lapack_api.cc
+++ b/src/operator/c_lapack_api.cc
@@ -98,6 +98,15 @@
     return 1; \
   }
 
+  #define MXNET_LAPACK_CWRAPPER11(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int nrhs, \
+                          dtype *a, int lda, dtype *b, int ldb, \
+                          dtype *s, dtype rcond, int *rank, \
+                          dtype *work, int lwork, int *iwork) { \
+    LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \
+    return 1; \
+  }
+
   #define MXNET_LAPACK_UNAVAILABLE(func) \
   int mxnet_lapack_##func(...) { \
     LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \
@@ -137,4 +146,7 @@
   MXNET_LAPACK_CWRAPPER9(sgesdd, float)
   MXNET_LAPACK_CWRAPPER9(dgesdd, double)
 
+  MXNET_LAPACK_CWRAPPER11(sgelsd, float)
+  MXNET_LAPACK_CWRAPPER11(dgelsd, double)
+
 #endif  // MSHADOW_USE_MKL == 0
diff --git a/src/operator/c_lapack_api.h b/src/operator/c_lapack_api.h
index 8b07265ba299..d3b844b704ab 100644
--- a/src/operator/c_lapack_api.h
+++ b/src/operator/c_lapack_api.h
@@ -190,13 +190,28 @@ extern "C" {
   #else
     #define MXNET_LAPACK_FSIG_GEEV(func, dtype) \
     void func##_(char *jobvl, char *jobvr, int *n, dtype *a, int *lda, \
-                dtype *wr, dtype *wi, \
-                dtype *vl, int *ldvl, dtype *vr, int *ldvr, \
-                dtype *work, int *lwork, int *info);
+                 dtype *wr, dtype *wi, \
+                 dtype *vl, int *ldvl, dtype *vr, int *ldvr, \
+                 dtype *work, int *lwork, int *info);
   #endif
 
   MXNET_LAPACK_FSIG_GEEV(sgeev, float)
   MXNET_LAPACK_FSIG_GEEV(dgeev, double)
+
+  #ifdef __ANDROID__
+    #define MXNET_LAPACK_FSIG_GELSD(func, dtype) \
+    int func##_(int *m, int *n, int *nrhs, dtype *a, int *lda, \
+                dtype *b, int *ldb, dtype *s, dtype *rcond, int *rank, \
+                dtype *work, int *lwork, int *iwork, int *info);
+  #else
+    #define MXNET_LAPACK_FSIG_GELSD(func, dtype) \
+    void func##_(int *m, int *n, int *nrhs, dtype *a, int *lda, \
+                 dtype *b, int *ldb, dtype *s, dtype *rcond, int *rank, \
+                 dtype *work, int *lwork, int *iwork, int *info);
+  #endif
+
+  MXNET_LAPACK_FSIG_GELSD(sgelsd, float)
+  MXNET_LAPACK_FSIG_GELSD(dgelsd, double)
 }
 
 #endif  // MSHADOW_USE_MKL == 0
@@ -362,6 +377,22 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAP_GEEV(s, float)
   MXNET_LAPACK_CWRAP_GEEV(d, double)
 
+  #define MXNET_LAPACK_CWRAP_GELSD(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##gelsd(int matrix_layout, int m, int n, int nrhs, \
+                                          dtype *a, int lda, dtype *b, int ldb, \
+                                          dtype *s, dtype rcond, int *rank, \
+                                          dtype *work, int lwork, int *iwork) { \
+    if (lwork != -1) { \
+      return LAPACKE_##prefix##gelsd(matrix_layout, m, n, nrhs, a, lda, b, ldb, \
+                                     s, rcond, rank); \
+    } \
+    *work = 0; \
+    *iwork = 0; \
+    return 0; \
+  }
+  MXNET_LAPACK_CWRAP_GELSD(s, float)
+  MXNET_LAPACK_CWRAP_GELSD(d, double)
+
 #elif MXNET_USE_LAPACK
 
   #define MXNET_LAPACK_ROW_MAJOR 101
@@ -577,6 +608,24 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAP_GESV(s, float)
   MXNET_LAPACK_CWRAP_GESV(d, double)
 
+  #define MXNET_LAPACK_CWRAP_GELSD(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##gelsd(int matrix_layout, int m, int n, int nrhs, \
+                                          dtype *a, int lda, dtype *b, int ldb, \
+                                          dtype *s, dtype rcond, int *rank, \
+                                          dtype *work, int lwork, int *iwork) { \
+    if (matrix_layout == MXNET_LAPACK_ROW_MAJOR) { \
+      CHECK(false) << "MXNET_LAPACK_" << #prefix << "gesv implemented for col-major layout only"; \
+      return 1; \
+    } else { \
+      int info(0); \
+      prefix##gelsd_(&m, &n, &nrhs, a, &lda, b, &ldb, s, &rcond, rank, \
+                     work, &lwork, iwork, &info); \
+      return info; \
+    } \
+  }
+  MXNET_LAPACK_CWRAP_GELSD(s, float)
+  MXNET_LAPACK_CWRAP_GELSD(d, double)
+
 #else
 
   #define MXNET_LAPACK_ROW_MAJOR 101
@@ -626,6 +675,12 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
                           dtype *vt, int ldvt, \
                           dtype *work, int lwork, int *iwork);
 
+  #define MXNET_LAPACK_CWRAPPER11(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int nrhs, \
+                          dtype *a, int lda, dtype *b, int ldb, \
+                          dtype *s, dtype rcond, int *rank, \
+                          dtype *work, int lwork, int *iwork);
+
   #define MXNET_LAPACK_UNAVAILABLE(func) \
   int mxnet_lapack_##func(...);
   MXNET_LAPACK_CWRAPPER1(spotrf, float)
@@ -662,6 +717,9 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAPPER9(sgesdd, float)
   MXNET_LAPACK_CWRAPPER9(dgesdd, double)
 
+  MXNET_LAPACK_CWRAPPER11(sgelsd, float)
+  MXNET_LAPACK_CWRAPPER11(dgelsd, double)
+
   #undef MXNET_LAPACK_CWRAPPER1
   #undef MXNET_LAPACK_CWRAPPER2
   #undef MXNET_LAPACK_CWRAPPER3
diff --git a/src/operator/numpy/linalg/np_lstsq-inl.h b/src/operator/numpy/linalg/np_lstsq-inl.h
new file mode 100644
index 000000000000..0389b7a5d92c
--- /dev/null
+++ b/src/operator/numpy/linalg/np_lstsq-inl.h
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2019 by Contributors
+ * \file np_lstsq-inl.h
+ * \brief Placeholder for lstlq
+ */
+#ifndef MXNET_OPERATOR_NUMPY_LINALG_NP_LSTSQ_INL_H_
+#define MXNET_OPERATOR_NUMPY_LINALG_NP_LSTSQ_INL_H_
+
+#include <mxnet/operator_util.h>
+#include <vector>
+#include <string>
+#include <utility>
+#include <algorithm>
+#include "../../operator_common.h"
+#include "../../mshadow_op.h"
+#include "../../c_lapack_api.h"
+
+namespace mxnet {
+namespace op {
+
+using namespace mshadow;
+
+struct LstsqParam : public dmlc::Parameter<LstsqParam> {
+  double rcond;
+  bool new_default;
+  DMLC_DECLARE_PARAMETER(LstsqParam) {
+    DMLC_DECLARE_FIELD(rcond)
+    .set_default(-1)
+    .describe("Cut-off ratio for small singular values");
+    DMLC_DECLARE_FIELD(new_default)
+    .set_default(false)
+    .describe("Specifies whether rcond is default which is machine precision");
+  }
+};
+
+template<typename xpu, typename DType>
+inline void linalg_gelsd_workspace_query(const int nrow,
+                                         const int ncol,
+                                         const int nrhs,
+                                         int *lwork,
+                                         int *liwork,
+                                         const Tensor<xpu, 2, DType>& A,
+                                         const Tensor<xpu, 2, DType>& B,
+                                         const Tensor<xpu, 1, DType>& S);
+
+template<typename xpu, typename DType>
+inline void linalg_gelsd(const int nrow,
+                         const int ncol,
+                         const int nrhs,
+                         const DType rcond,
+                         int *rank,
+                         const Tensor<xpu, 2, DType>& A,
+                         const Tensor<xpu, 2, DType>& B,
+                         const Tensor<xpu, 1, DType>& SingularValues,
+                         const Tensor<xpu, 1, DType>& Work,
+                         const Tensor<xpu, 1, int>& Iwork);
+
+struct LstsqTypeTransposeHelper {
+  template<typename InDType, typename OutDType>
+  MSHADOW_XINLINE static void Map(int i, const InDType *in_ptr, OutDType *out_ptr,
+                                  const int nrow, const int ncol, const int ld) {
+    if (ld >= nrow && i < nrow * ncol) {
+      out_ptr[i / ncol + (i % ncol) * ld] = static_cast<OutDType>(in_ptr[i]);
+    }
+  }
+};
+
+template<int req>
+struct ValuesAssignHelper {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *in_data, DType *out_data) {
+    KERNEL_ASSIGN(out_data[i], req, in_data[i]);
+  }
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType& in_data, DType *out_data) {
+    KERNEL_ASSIGN(out_data[i], req, in_data);
+  }
+};
+
+template<int req>
+struct SolutionAssignHelper {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *trans_b_ptr, DType *x_ptr,
+                                  const int nrow, const int ncol, const int nrhs, const int ldb) {
+    if (i < ncol * nrhs && ldb >= nrow && ldb >= ncol) {
+      KERNEL_ASSIGN(x_ptr[i], req, trans_b_ptr[i / nrhs  + (i % nrhs) * ldb]);
+    }
+  }
+};
+
+template<int req>
+struct ResidualsAssignHelper {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *trans_b_ptr, DType *residuals_ptr,
+                                  const int nrow, const int ncol, const int nrhs, const int ldb) {
+    if (i < nrhs) {
+      DType residuals_values = 0;
+      for (int j = ncol; j < nrow; ++j) {
+        residuals_values += trans_b_ptr[j + i * ldb] * trans_b_ptr[j + i * ldb];
+      }
+      KERNEL_ASSIGN(residuals_ptr[i], req, residuals_values);
+    }
+  }
+};
+
+#define LINALG_CPU_GELSD_WORKSPACE_QUERY(func, DType) \
+template<> inline void \
+linalg_gelsd_workspace_query<cpu, DType>(const int nrow, \
+                                         const int ncol, \
+                                         const int nrhs, \
+                                         int *lwork, \
+                                         int *liwork, \
+                                         const Tensor<cpu, 2, DType>& A, \
+                                         const Tensor<cpu, 2, DType>& B, \
+                                         const Tensor<cpu, 1, DType>& S) { \
+  CHECK(A.size(0) == ncol && A.size(1) == A.stride_) \
+    << "ncol or lda dismatch A shape and lda should >= max(1, nrow)."; \
+  CHECK(B.size(0) == nrhs && B.size(1) == B.stride_) \
+    << "nrhs or ldb dismatch B shape and ldb should >= max(1, max(nrow, ncol))"; \
+  DType temp_work = -1, rcond = -1; \
+  int temp_iwork = -1, rank = -1; \
+  int info = MXNET_LAPACK_##func(MXNET_LAPACK_COL_MAJOR, nrow, ncol, nrhs, \
+                                 A.dptr_, static_cast<int>(A.stride_), \
+                                 B.dptr_, static_cast<int>(B.stride_), \
+                                 B.dptr_, rcond, &rank, \
+                                 &temp_work, -1, &temp_iwork); \
+  CHECK_GE(info, 0) << "MXNET_LAPACK_" << #func << ": " \
+    << "the " << -info << "-th argument had an illegal value"; \
+  *lwork = static_cast<int>(temp_work); \
+  *liwork = temp_iwork; \
+  return; \
+}
+
+#define LINALG_CPU_GELSD(func, DType) \
+template<> inline void \
+linalg_gelsd<cpu, DType>(const int nrow, \
+                         const int ncol, \
+                         const int nrhs, \
+                         const DType rcond, \
+                         int *rank, \
+                         const Tensor<cpu, 2, DType>& A, \
+                         const Tensor<cpu, 2, DType>& B, \
+                         const Tensor<cpu, 1, DType>& SingularValues, \
+                         const Tensor<cpu, 1, DType>& Work, \
+                         const Tensor<cpu, 1, int>& Iwork) { \
+  CHECK(A.size(0) == ncol && A.size(1) == A.stride_) \
+    << "ncol or lda dismatch A shape and lda should >= max(1, nrow)."; \
+  CHECK(B.size(0) == nrhs && B.size(1) == B.stride_) \
+    << "nrhs or ldb dismatch B shape and ldb should >= max(1, max(nrow, ncol))"; \
+  CHECK(SingularValues.MSize() >= std::min(nrow, ncol)) \
+    << "SingularValues is too small"; \
+  const int lwork = Work.MSize(); \
+  int info = MXNET_LAPACK_##func(MXNET_LAPACK_COL_MAJOR, nrow, ncol, nrhs, \
+                                 A.dptr_, A.stride_, B.dptr_, B.stride_, \
+                                 SingularValues.dptr_, rcond, rank, \
+                                 Work.dptr_, lwork, Iwork.dptr_); \
+  CHECK_GE(info, 0) << "MXNET_LAPACK_" << #func << ": " \
+    << "the " << -info << "-th argument had an illegal value"; \
+  CHECK_LE(info, 0) << "MXNET_LAPACK_" << #func << ": " \
+    << "the algorithm for computing the SVD failed to converge; " \
+    << info << " off-diagonal elements of an intermediate bidiagonal " \
+    << "form did not converge to zero."; \
+  return; \
+}
+
+LINALG_CPU_GELSD_WORKSPACE_QUERY(sgelsd, float)
+LINALG_CPU_GELSD_WORKSPACE_QUERY(dgelsd, double)
+
+LINALG_CPU_GELSD(sgelsd, float)
+LINALG_CPU_GELSD(dgelsd, double)
+
+#ifdef __CUDACC__
+
+#define LINALG_GPU_GELSD_WORKSPACE_QUERY(DType) \
+template<> inline void \
+linalg_gelsd_workspace_query<gpu, DType>(const int nrow, \
+                                         const int ncol, \
+                                         const int nrhs, \
+                                         int *lwork, \
+                                         int *liwork, \
+                                         const Tensor<gpu, 2, DType>& A, \
+                                         const Tensor<gpu, 2, DType>& B, \
+                                         const Tensor<gpu, 1, DType>& S) { \
+  LOG(FATAL) << "linalg gesld workspace query is unsupported in gpu!"; \
+  return; \
+}
+
+#define LINALG_GPU_GELSD(DType) \
+template<> inline void \
+linalg_gelsd<gpu, DType>(const int nrow, \
+                         const int ncol, \
+                         const int nrhs, \
+                         const DType rcond, \
+                         int *rank, \
+                         const Tensor<gpu, 2, DType>& A, \
+                         const Tensor<gpu, 2, DType>& B, \
+                         const Tensor<gpu, 1, DType>& SingularValues, \
+                         const Tensor<gpu, 1, DType>& Work, \
+                         const Tensor<gpu, 1, int>& Iwork) { \
+  LOG(FATAL) << "linalg gesld is unsupported in gpu!"; \
+  return; \
+}
+
+LINALG_GPU_GELSD_WORKSPACE_QUERY(float)
+LINALG_GPU_GELSD_WORKSPACE_QUERY(double)
+
+LINALG_GPU_GELSD(float)
+LINALG_GPU_GELSD(double)
+
+#endif  // __CUDACC__
+
+inline bool GetOutputShapes(const mxnet::TShape& a_shape,
+                            const mxnet::TShape& b_shape,
+                            mxnet::ShapeVector *out_attrs) {
+  if (!ndim_is_known(a_shape) || !ndim_is_known(b_shape)) {
+    return false;
+  }
+  const int a_ndim = a_shape.ndim();
+  const int b_ndim = b_shape.ndim();
+  const int a_nrow = a_shape[0];
+  const int a_ncol = a_shape[1];
+  const int b_nrow = b_shape[0];
+  const int b_nrhs = b_ndim == 2 ? b_shape[1] : 1;
+  CHECK_EQ(a_ndim, 2) << a_ndim
+    << "-dimensional array given. Array must be two-dimensional";
+  CHECK(b_ndim == 1 || b_ndim == 2) << b_ndim
+    << "-dimensional array given. Array must be one-dimensional or two-dimensional";
+  CHECK_EQ(a_nrow, b_nrow)
+    << "Incompatible dimensions of inputs";
+  // x_shape
+  if (b_ndim == 2) {
+    std::vector<int> x_shape_vec({a_ncol, b_nrhs});
+    SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(x_shape_vec.begin(), x_shape_vec.end()));
+  } else {
+    SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(1, a_ncol));
+  }
+  // temp_residuals_shape
+  SHAPE_ASSIGN_CHECK(*out_attrs, 1, mxnet::TShape(1, static_cast<dim_t>(std::max(1, b_nrhs))));
+  // rank_shape
+  SHAPE_ASSIGN_CHECK(*out_attrs, 2, mxnet::TShape(0, 0));
+  // s_shape
+  if (a_nrow == 0 || a_ncol == 0) {
+    SHAPE_ASSIGN_CHECK(*out_attrs, 3, mxnet::TShape(1, 0));
+  } else {
+    SHAPE_ASSIGN_CHECK(*out_attrs, 3, mxnet::TShape(1, std::max(1, std::min(a_nrow, a_ncol))));
+  }
+  return shape_is_known(*out_attrs);
+}
+
+template<typename xpu, typename DType>
+size_t LstsqWorkspaceSize(const TBlob& a,
+                          const TBlob& b,
+                          const mxnet::TShape& x_shape,
+                          int *lwork,
+                          int *liwork,
+                          const OpContext& ctx) {
+  const int a_ndim = a.ndim();
+  const int b_ndim = b.ndim();
+  CHECK(a_ndim == 2 && (b_ndim == 2 || b_ndim == 1)) << "Wrong array ndim";
+  CHECK_EQ(a.shape_[0], b.shape_[0]) << "Inputs shape dismatch";
+
+  size_t workspace_size = 0;
+  Stream<cpu> *s_cpu = ctx.get_stream<cpu>();
+  MSHADOW_SGL_DBL_TYPE_SWITCH(a.type_flag_, AType, {
+    MSHADOW_SGL_DBL_TYPE_SWITCH(b.type_flag_, BType, {
+      const mxnet::TShape& a_shape = a.shape_;
+      const mxnet::TShape& b_shape = b.shape_;
+      if (xpu::kDevCPU) {
+        int nrow = a_shape[0];
+        int ncol = a_shape[1];
+        int nrhs = b_ndim == 2 ? b_shape[1] : 1;
+        int lda = std::max(1, nrow);
+        int ldb = std::max(1, std::max(nrow, ncol));
+        // Lapack routine can't handle lda = 0 and ldb = 0.
+        // If nrow == 0, leading dimension of A_trans and B_trans will be 0.
+        if (nrow == 0) { return 0U; }
+        if (ncol == 0 && nrhs == 0) { return 0U; }
+        // If ncol != 0, need to invoke lapack routine.
+        // Lapack routine can't handle n_rhs = 0, so allocate the array one larger in that axis.
+        int temp_nrhs = nrhs == 0 ? 1 : nrhs;
+        std::vector<DType> temp_a_vec(ncol * lda, 0);
+        std::vector<DType> temp_b_vec(temp_nrhs * ldb, 0);
+        std::vector<DType> temp_s_vec(std::max(1, std::min(nrow, ncol)), 0);
+        mshadow::Tensor<cpu, 2, DType> A(temp_a_vec.data(), Shape2(ncol, lda), lda, s_cpu);
+        mshadow::Tensor<cpu, 2, DType> B(temp_b_vec.data(), Shape2(temp_nrhs, ldb), ldb, s_cpu);
+        mshadow::Tensor<cpu, 1, DType> S(temp_s_vec.data(), Shape1(temp_s_vec.size()), s_cpu);
+        // Invoke lapack workspace query.
+        linalg_gelsd_workspace_query<cpu, DType>(nrow, ncol, temp_nrhs,
+                                                 lwork, liwork, A, B, S);
+        // For A size because on lapack routine exit, A will be overwritten.
+        workspace_size += ncol * lda * sizeof(DType);
+        // For B size because on lapack routine exit, B will be overwritten by solution result.
+        workspace_size += temp_nrhs * ldb * sizeof(DType);
+        // For singular values size.
+        workspace_size += std::max(1, std::min(nrow, ncol)) * sizeof(DType);
+        // For workspace size in linalg_gesld.
+        workspace_size += (*lwork) * sizeof(DType) + (*liwork) * sizeof(int);
+      }
+    });
+  });
+  return workspace_size;
+}
+
+template<typename xpu>
+void LstsqOpForwardImpl(const TBlob& a,
+                        const TBlob& b,
+                        const TBlob& x,
+                        const TBlob& temp_residuals,
+                        const TBlob& rank,
+                        const TBlob& singularValues,
+                        bool *empty_residuals,
+                        const int& lwork,
+                        const int& liwork,
+                        std::vector<char> *workspace,
+                        const nnvm::NodeAttrs& attrs,
+                        const OpContext& ctx,
+                        const std::vector<OpReqType>& req) {
+  // Get param.
+  double rcond = nnvm::get<LstsqParam>(attrs.parsed).rcond;
+  bool new_default = nnvm::get<LstsqParam>(attrs.parsed).new_default;
+  if (new_default) {
+    rcond *= std::max(a.shape_[0], a.shape_[1]);
+  }
+  const mxnet::TShape& a_shape = a.shape_;
+  const mxnet::TShape& b_shape = b.shape_;
+  MSHADOW_SGL_DBL_TYPE_SWITCH(x.type_flag_, DType, {
+    mshadow::Stream<xpu> *s = ctx.get_stream<xpu>();
+    const int nrow = a_shape[0];
+    const int ncol = a_shape[1];
+    const int nrhs = b.ndim() == 2 ? b_shape[1] : 1;
+    const int lda = std::max(1, nrow);
+    const int ldb = std::max(1, std::max(nrow, ncol));
+    const int snum = std::max(1, std::min(nrow, ncol));
+    if (nrow == 0) {
+      // Assign 0 for all values in x.
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        mxnet_op::Kernel<ValuesAssignHelper<req_type>, xpu>::Launch(
+          s, x.Size(), static_cast<DType>(0), x.dptr<DType>());
+      });
+      // Assign values for rank.
+      ASSIGN_DISPATCH(*rank.dptr<int>(), kWriteTo, 0);
+      // Assign values for empty_residuals.
+      *empty_residuals = true;
+      return;
+    }
+    if (ncol == 0 && nrhs == 0) {
+      // Assign values for rank.
+      ASSIGN_DISPATCH(*rank.dptr<int>(), kWriteTo, 0);
+      // Assign values for empty_residuals.
+      *empty_residuals = true;
+      return;
+    }
+    int temp_nrhs = nrhs == 0 ? 1 : nrhs;
+    mxnet::TShape trans_a_shape(mxnet::Tuple<dim_t>({ ncol, lda }));
+    mxnet::TShape trans_b_shape(mxnet::Tuple<dim_t>({ temp_nrhs, ldb }));
+    // Allocate data memory.
+    DType *a_ptr = reinterpret_cast<DType*>(workspace->data());
+    DType *b_ptr = a_ptr + trans_a_shape.Size();
+    DType *s_ptr = b_ptr + trans_b_shape.Size();
+    DType *work_ptr = s_ptr + snum;
+    int *iwork_ptr = reinterpret_cast<int*>(work_ptr + lwork);
+    TBlob trans_a(a_ptr, trans_a_shape, a.dev_mask(), a.dev_id());
+    TBlob trans_b(b_ptr, trans_b_shape, b.dev_mask(), b.dev_id());
+    TBlob singular_values(s_ptr, Shape1(snum), singularValues.dev_mask(), singularValues.dev_id());
+    TBlob work(work_ptr, Shape1(lwork), x.dev_mask(), x.dev_id());
+    TBlob iwork(iwork_ptr, Shape1(liwork), x.dev_mask(), x.dev_id());
+    // Transpose a to trans_a.
+    MSHADOW_SGL_DBL_TYPE_SWITCH(a.type_flag_, AType, {
+      mxnet_op::Kernel<LstsqTypeTransposeHelper, xpu>::Launch(s, a.Size(), a.dptr<AType>(),
+                                                              trans_a.dptr<DType>(),
+                                                              nrow, ncol, lda);
+    });
+    // If nrhs == 0, assign 0 to trans_b directly.
+    // If nrhs != 0, assign 0 to initialize trans_b because trans_b.Size > b.Size when nrow < ncol.
+    mxnet_op::Kernel<ValuesAssignHelper<kWriteTo>, xpu>::Launch(
+      s, trans_b.Size(), static_cast<DType>(0), trans_b.dptr<DType>());
+    if (nrhs != 0) {
+      // Transpose b to trans_b.
+      MSHADOW_SGL_DBL_TYPE_SWITCH(b.type_flag_, BType, {
+        mxnet_op::Kernel<LstsqTypeTransposeHelper, xpu>::Launch(s, b.Size(), b.dptr<BType>(),
+                                                                trans_b.dptr<DType>(),
+                                                                nrow, nrhs, ldb);
+      });
+    }
+    // Invoke lapack routines.
+    linalg_gelsd<xpu, DType>(nrow, ncol, temp_nrhs,
+                             rcond, rank.dptr<int>(),
+                             trans_a.get<xpu, 2, DType>(s),
+                             trans_b.get<xpu, 2, DType>(s),
+                             singular_values.get<xpu, 1, DType>(s),
+                             work.get<xpu, 1, DType>(s),
+                             iwork.get<xpu, 1, int>(s));
+    if (ncol != 0 && nrhs == 0) {
+      // Assign values for singularValues.
+      MXNET_ASSIGN_REQ_SWITCH(req[3], req_type, {
+        mxnet_op::Kernel<ValuesAssignHelper<req_type>, xpu>::Launch(
+          s, singularValues.Size(), singular_values.dptr<DType>(), singularValues.dptr<DType>());
+      });
+      // Assign values for empty_residuals.
+      *empty_residuals = true;
+      return;
+    } else {
+      // Assign values for x.
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        mxnet_op::Kernel<SolutionAssignHelper<req_type>, xpu>::Launch(
+          s, x.Size(), trans_b.dptr<DType>(), x.dptr<DType>(), nrow, ncol, nrhs, ldb);
+      });
+      // Assign values for residuals and residualsEmpty.
+      if (*(rank.dptr<int>()) < ncol || nrow <= ncol) {
+        *empty_residuals = true;
+      } else {
+        *empty_residuals = false;
+        MXNET_ASSIGN_REQ_SWITCH(req[1], req_type, {
+          mxnet_op::Kernel<ResidualsAssignHelper<req_type>, xpu>::Launch(
+            s, temp_residuals.Size(), trans_b.dptr<DType>(),
+            temp_residuals.dptr<DType>(), nrow, ncol, nrhs, ldb);
+        });
+      }
+      // Assign values for singularValues.
+      MXNET_ASSIGN_REQ_SWITCH(req[3], req_type, {
+        mxnet_op::Kernel<ValuesAssignHelper<req_type>, xpu>::Launch(
+          s, singularValues.Size(), singular_values.dptr<DType>(), singularValues.dptr<DType>());
+      });
+    }
+  });
+}
+
+template<typename DType>
+inline void GpuCallbackCpuImpl(const TBlob& a,
+                               const TBlob& b,
+                               const TBlob& x,
+                               const TBlob& rank,
+                               const TBlob& singularValues,
+                               const NDArray& residuals_ndarray,
+                               const mxnet::TShape& temp_residuals_shape,
+                               const int& lwork,
+                               const int& liwork,
+                               std::vector<char> *workspace,
+                               const nnvm::NodeAttrs& attrs,
+                               const OpContext& ctx,
+                               const std::vector<OpReqType>& req) {
+#if MXNET_USE_CUDA
+  MSHADOW_SGL_DBL_TYPE_SWITCH(a.type_flag_, AType, {
+    MSHADOW_SGL_DBL_TYPE_SWITCH(b.type_flag_, BType, {
+      std::vector<AType> a_vec(a.Size(), 0);
+      std::vector<BType> b_vec(b.Size(), 0);
+      std::vector<DType> x_vec(x.Size(), 0);
+      std::vector<DType> temp_residuals_vec(temp_residuals_shape.Size(), 0);
+      std::vector<int> rank_vec(rank.Size(), 0);
+      std::vector<DType> singularValues_vec(singularValues.Size(), 0);
+      mshadow::Stream<gpu> *s = ctx.get_stream<gpu>();
+      cudaStream_t stream = Stream<gpu>::GetStream(s);
+      // Copy inputs from gpu to cpu.
+      CUDA_CALL(cudaMemcpyAsync(a_vec.data(), a.dptr<AType>(), sizeof(AType) * a.Size(),
+                                cudaMemcpyDeviceToHost, stream));
+      CUDA_CALL(cudaMemcpyAsync(b_vec.data(), b.dptr<BType>(), sizeof(BType) * b.Size(),
+                                cudaMemcpyDeviceToHost, stream));
+      CUDA_CALL(cudaStreamSynchronize(stream));
+      mxnet::TBlob a_data(a_vec.data(), a.shape_, cpu::kDevMask, -1);
+      mxnet::TBlob b_data(b_vec.data(), b.shape_, cpu::kDevMask, -1);
+      mxnet::TBlob x_data(x_vec.data(), x.shape_, cpu::kDevMask, -1);
+      mxnet::TBlob rank_data(rank_vec.data(), rank.shape_, cpu::kDevMask, -1);
+      mxnet::TBlob temp_residuals_data(temp_residuals_vec.data(),
+                                       temp_residuals_shape, cpu::kDevMask, -1);
+      mxnet::TBlob singularValues_data(singularValues_vec.data(),
+                                       singularValues.shape_, cpu::kDevMask, -1);
+      // Op forward implement on cpu.
+      bool empty_residuals = false;
+      LstsqOpForwardImpl<cpu>(a_data, b_data, x_data,
+                              temp_residuals_data, rank_data, singularValues_data,
+                              &empty_residuals, lwork, liwork, workspace, attrs, ctx, req);
+      if (empty_residuals) {
+        // Set residuals to empty deriectly.
+        const_cast<NDArray&>(residuals_ndarray).Init(mxnet::TShape(1, 0));
+      } else {
+        // No need set residuals to empty.
+        const_cast<NDArray&>(residuals_ndarray).Init(temp_residuals_shape);
+        // Copy back to gpu.
+        CUDA_CALL(cudaMemcpyAsync(residuals_ndarray.data().dptr<DType>(), temp_residuals_vec.data(),
+                                  sizeof(DType) * temp_residuals_data.Size(),
+                                  cudaMemcpyHostToDevice, stream));
+      }
+      CUDA_CALL(cudaStreamSynchronize(stream));
+      // Copy back to gpu.
+      CUDA_CALL(cudaMemcpyAsync(x.dptr<DType>(), x_vec.data(), sizeof(DType) * x.Size(),
+                                cudaMemcpyHostToDevice, stream));
+      CUDA_CALL(cudaMemcpyAsync(rank.dptr<int>(), rank_vec.data(), sizeof(int) * rank.Size(),
+                                cudaMemcpyHostToDevice, stream));
+      CUDA_CALL(cudaMemcpyAsync(singularValues.dptr<DType>(), singularValues_vec.data(),
+                                sizeof(DType) * singularValues.Size(),
+                                cudaMemcpyHostToDevice, stream));
+      CUDA_CALL(cudaStreamSynchronize(stream));
+    });
+  });
+#else
+  LOG(FATAL) << "Please build with USE_CUDA=1 to enable GPU";
+#endif  // MXNET_USE_CUDA
+}
+
+template<typename xpu>
+void LstsqOpForward(const nnvm::NodeAttrs& attrs,
+                    const OpContext& ctx,
+                    const std::vector<NDArray>& inputs,
+                    const std::vector<OpReqType>& req,
+                    const std::vector<NDArray>& outputs) {
+  CHECK_EQ(inputs.size(), 2U);
+  CHECK_EQ(outputs.size(), 4U);
+  CHECK_EQ(req.size(), 4U);
+  CHECK(req[0] == kWriteTo || req[0] == kWriteInplace);
+  CHECK(req[1] == kWriteTo || req[1] == kWriteInplace);
+  CHECK(req[2] == kWriteTo || req[2] == kWriteInplace);
+  CHECK(req[3] == kWriteTo || req[3] == kWriteInplace);
+  using namespace mshadow;
+  const NDArray& a_ndarray = inputs[0];
+  const NDArray& b_ndarray = inputs[1];
+  const NDArray& x_ndarray = outputs[0];
+  const NDArray& residuals_ndarray = outputs[1];
+  const NDArray& rank_ndarray = outputs[2];
+  const NDArray& singularValues_ndarray = outputs[3];
+  const mxnet::TShape& a_shape = a_ndarray.shape();
+  const mxnet::TShape& b_shape = b_ndarray.shape();
+  // Force set output shapes.
+  mxnet::ShapeVector out_shapes(4);
+  GetOutputShapes(a_shape, b_shape, &out_shapes);
+  const mxnet::TShape& x_shape = out_shapes[0];
+  const mxnet::TShape& temp_residuals_shape = out_shapes[1];
+  const mxnet::TShape& rank_shape = out_shapes[2];
+  const mxnet::TShape& singularValues_shape = out_shapes[3];
+
+  MSHADOW_SGL_DBL_TYPE_SWITCH(x_ndarray.dtype(), DType, {
+    // Allocate workspace.
+    int lwork = 0, liwork = 0;
+    size_t workspace_size = LstsqWorkspaceSize<cpu, DType>(a_ndarray.data(), b_ndarray.data(),
+                                                           x_shape, &lwork, &liwork, ctx);
+    std::vector<char> workspace(workspace_size);
+    // Force init.
+    const_cast<NDArray&>(x_ndarray).Init(x_shape);
+    const_cast<NDArray&>(rank_ndarray).Init(rank_shape);
+    const_cast<NDArray&>(singularValues_ndarray).Init(singularValues_shape);
+
+    // Allocate temp space for residuals
+    std::vector<DType> temp_residuals_vec(temp_residuals_shape.Size(), 0);
+    if (xpu::kDevCPU) {
+      bool empty_residuals = false;
+      TBlob temp_residuals(temp_residuals_vec.data(),
+                           Shape1(temp_residuals_vec.size()), cpu::kDevMask, -1);
+      LstsqOpForwardImpl<cpu>(a_ndarray.data(), b_ndarray.data(), x_ndarray.data(),
+                              temp_residuals, rank_ndarray.data(), singularValues_ndarray.data(),
+                              &empty_residuals, lwork, liwork, &workspace, attrs, ctx, req);
+      if (empty_residuals) {
+        // Set residuals to empty deriectly.
+        const_cast<NDArray&>(residuals_ndarray).Init(mxnet::TShape(1, 0));
+      } else {
+        // No need set residuals to empty.
+        const_cast<NDArray&>(residuals_ndarray).Init(temp_residuals.shape_);
+        MXNET_ASSIGN_REQ_SWITCH(req[1], req_type, {
+          mxnet_op::Kernel<ValuesAssignHelper<req_type>, cpu>::Launch(
+            ctx.get_stream<cpu>(), temp_residuals.Size(), temp_residuals.dptr<DType>(),
+            residuals_ndarray.data().dptr<DType>());
+        });
+      }
+    } else {
+      GpuCallbackCpuImpl<DType>(a_ndarray.data(), b_ndarray.data(), x_ndarray.data(),
+                                rank_ndarray.data(), singularValues_ndarray.data(),
+                                residuals_ndarray, temp_residuals_shape,
+                                lwork, liwork, &workspace, attrs, ctx, req);
+    }
+  });
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_LINALG_NP_LSTSQ_INL_H_
diff --git a/src/operator/numpy/linalg/np_lstsq.cc b/src/operator/numpy/linalg/np_lstsq.cc
new file mode 100644
index 000000000000..91bbaa7b9633
--- /dev/null
+++ b/src/operator/numpy/linalg/np_lstsq.cc
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2019 by Contributors
+ * \file np_lstsq.cc
+ * \brief CPU implementation of the lstsq Operator
+ */
+#include "./np_lstsq-inl.h"
+
+namespace mxnet {
+namespace op {
+
+inline bool LstsqOpStorageType(const nnvm::NodeAttrs& attrs,
+                               const int dev_mask,
+                               DispatchMode* dispatch_mode,
+                               std::vector<int> *in_attrs,
+                               std::vector<int> *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  for (int& attr : *in_attrs) {
+    CHECK_EQ(attr, kDefaultStorage) << "Only default storage is supported";
+  }
+  for (int& attr : *out_attrs) {
+    attr = kDefaultStorage;
+  }
+  *dispatch_mode = DispatchMode::kFComputeEx;
+  return true;
+}
+
+inline bool LstsqOpType(const nnvm::NodeAttrs& attrs,
+                        std::vector<int>* in_attrs,
+                        std::vector<int>* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 4U);
+  const int a_type = in_attrs->at(0);
+  const int b_type = in_attrs->at(1);
+  CHECK(a_type == mshadow::kFloat32 || a_type == mshadow::kFloat64)
+    << "lstsq operation only supports 32-bit and 64-bit floating point";
+  CHECK(b_type == mshadow::kFloat32 || b_type == mshadow::kFloat64)
+    << "lstsq operation only supports 32-bit and 64-bit floating point";
+
+  if (mshadow::kFloat32 == a_type && mshadow::kFloat32 == b_type) {
+    TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kFloat32);
+    TYPE_ASSIGN_CHECK(*out_attrs, 1, mshadow::kFloat32);
+    TYPE_ASSIGN_CHECK(*out_attrs, 2, mshadow::kInt32);
+    TYPE_ASSIGN_CHECK(*out_attrs, 3, mshadow::kFloat32);
+  } else {
+    TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kFloat64);
+    TYPE_ASSIGN_CHECK(*out_attrs, 1, mshadow::kFloat64);
+    TYPE_ASSIGN_CHECK(*out_attrs, 2, mshadow::kInt32);
+    TYPE_ASSIGN_CHECK(*out_attrs, 3, mshadow::kFloat64);
+  }
+  return out_attrs->at(0) != -1 &&
+         out_attrs->at(1) != -1 &&
+         out_attrs->at(2) != -1 &&
+         out_attrs->at(3) != -1;
+}
+
+DMLC_REGISTER_PARAMETER(LstsqParam);
+
+NNVM_REGISTER_OP(_npi_lstsq)
+.describe(R"code()code" ADD_FILELINE)
+.set_attr_parser(mxnet::op::ParamParser<LstsqParam>)
+.set_num_inputs(2)
+.set_num_outputs(4)
+.set_attr<nnvm::FListInputNames>("FListInputNames", [](const NodeAttrs& attrs){
+  return std::vector<std::string>{"A", "B"};
+})
+.set_attr<nnvm::FInferType>("FInferType", LstsqOpType)
+.set_attr<FInferStorageType>("FInferStorageType", LstsqOpStorageType)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs){
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.set_attr<FComputeEx>("FComputeEx<cpu>", LstsqOpForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("A", "NDArray-or-Symbol", "Tensor of matrix")
+.add_argument("B", "NDArray-or-Symbol", "Tensor of matrix")
+.add_arguments(LstsqParam::__FIELDS__());
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_lstsq.cu b/src/operator/numpy/linalg/np_lstsq.cu
new file mode 100644
index 000000000000..df8d7869f563
--- /dev/null
+++ b/src/operator/numpy/linalg/np_lstsq.cu
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_lstsq.cu
+ * \brief GPU implementation placeholder of lstsq operator
+ */
+
+#include <mxnet/operator_util.h>
+#include "./np_lstsq-inl.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_lstsq)
+.set_attr<FComputeEx>("FComputeEx<gpu>", LstsqOpForward<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 2d1710eeedc9..c4e4b599e2d6 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -626,6 +626,36 @@ def _add_workload_linalg_pinv():
             OpArgMngr.add_workload('linalg.pinv', np.array(a_np, dtype=dtype), np.array(rcond_np, dtype=dtype), hermitian)
 
 
+def _add_workload_linalg_lstsq():
+    shapes = [
+        ((0, 0), (0,)),
+        ((0, 0), (0, 0)),
+        ((4, 0), (4,)),
+        ((4, 0), (4, 2)),
+        ((0, 2), (0, 4)),
+        ((4, 2), (4, 0)),
+        ((0, 0), (0, 4)),
+        ((0, 2), (0, 0)),
+        ((4, 0), (4, 0)),
+        ((4, 2), (4,)),
+        ((4, 2), (4, 3)),
+        ((4, 6), (4, 3)),
+    ]
+    rconds = [None, "random", "warn"]
+    dtypes = (np.float32, np.float64)
+    for dtype, rcond in itertools.product(dtypes, rconds):
+        for a_shape, b_shape in shapes:
+            if rcond == "random":
+                rcond = _np.random.uniform(100, 200)
+            if rcond == "warn":
+                rcond = -1
+            a_np = _np.random.uniform(-10.0, 10.0, a_shape)
+            b_np = _np.random.uniform(-10.0, 10.0, b_shape)
+            a = np.array(a_np, dtype=dtype)
+            b = np.array(b_np, dtype=dtype)
+            OpArgMngr.add_workload('linalg.lstsq', a, b, rcond)
+
+
 def _add_workload_linalg_eigvals():
     OpArgMngr.add_workload('linalg.eigvals', np.array(_np.diag((0, 0)), dtype=np.float64))
     OpArgMngr.add_workload('linalg.eigvals', np.array(_np.diag((1, 1)), dtype=np.float64))
@@ -2002,15 +2032,6 @@ def _add_workload_linalg_cond():
     OpArgMngr.add_workload('linalg.cond', A, 'fro')
 
 
-def _add_workload_linalg_lstsq():
-    y = np.array([-1, 0.2, 0.9, 2.1])
-    A = np.array([[ 0.,  1.],
-                  [ 1.,  1.],
-                  [ 2.,  1.],
-                  [ 3.,  1.]])
-    OpArgMngr.add_workload('linalg.lstsq', A, y, rcond=None)
-
-
 def _add_workload_linalg_matrix_power():
     i = np.array([[0, 1], [-1, 0]])
     OpArgMngr.add_workload('linalg.matrix_power', i, 3)
@@ -2797,6 +2818,7 @@ def _prepare_workloads():
     _add_workload_linalg_det()
     _add_workload_linalg_tensorinv()
     _add_workload_linalg_tensorsolve()
+    _add_workload_linalg_lstsq()
     _add_workload_linalg_pinv()
     _add_workload_linalg_eigvals()
     _add_workload_linalg_eig()
@@ -2804,7 +2826,6 @@ def _prepare_workloads():
     _add_workload_linalg_eigh()
     _add_workload_linalg_slogdet()
     _add_workload_linalg_cond()
-    _add_workload_linalg_lstsq()
     _add_workload_linalg_matrix_power()
     _add_workload_linalg_matrix_rank()
     _add_workload_linalg_multi_dot()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 5bc3923aa79c..3242a147eff6 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -5530,6 +5530,82 @@ def newInvertibleMatrix_2D(shape, max_cond=4):
                 check_tensorsolve(mx_out, a.asnumpy(), b.asnumpy(), axes)
 
 
+@with_seed()
+@use_np
+def test_np_linalg_lstsq():
+    class TestLstsq(HybridBlock):
+        def __init__(self, rcond):
+            super(TestLstsq, self).__init__()
+            self._rcond = rcond
+        
+        def hybrid_forward(self, F, a, b, rcond='warn'):
+            return F.np.linalg.lstsq(a, b, rcond=self._rcond)
+
+    def check_lstsq(a_np, b_np, rcond_np, x, residuals, rank, s):
+        try:
+            if rcond_np == 'warn':
+                rcond_np = -1
+            x_expected, residuals_expected, rank_expected, s_expected = _np.linalg.lstsq(a_np, b_np, rcond_np)
+        except Exception as e:
+            print("a:", a_np)
+            print("a shape:", a_np.shape)
+            print("b:", b_np)
+            print("b shape:", b_np.shape)
+            print(e)
+        else:
+            assert x.shape == x_expected.shape
+            assert residuals.shape == residuals_expected.shape
+            assert rank.shape == rank_expected.shape
+            assert s.shape == s_expected.shape
+            assert_almost_equal(x.asnumpy(), x_expected, rtol=rtol, atol=atol)
+            assert_almost_equal(residuals.asnumpy(), residuals_expected, rtol=rtol, atol=atol)
+            assert_almost_equal(rank.asnumpy(), rank_expected, rtol=rtol, atol=atol)
+            assert_almost_equal(s.asnumpy(), s_expected, rtol=rtol, atol=atol)
+
+    shapes = [
+        ((4, 0), (4,)),   # ncol == 0
+        ((4, 0), (4, 2)), # ncol == 0
+        ((0, 2), (0,)),   # nrow == 0
+        ((0, 2), (0, 4)), # nrow == 0
+        ((4, 2), (4, 0)), # nrhs == 0
+        ((4, 4), (4, 0)), # nrhs == 0
+        ((4, 6), (4, 0)), # nrhs == 0
+        ((0, 0), (0, 4)), # nrow == 0, ncol == 0
+        ((0, 2), (0, 0)), # nrow == 0, nrhs == 0
+        ((4, 0), (4, 0)), # ncol == 0, nrhs == 0
+        ((0, 0), (0,)),   # nrow == 0, ncol == 0, nrhs = none
+        ((0, 0), (0, 0)), # nrow == 0, ncol == 0, nrhs = 0
+        ((2, 1), (2,)),
+        ((4, 1), (4,)),
+        ((4, 2), (4,)),
+        ((4, 4), (4,)),
+        ((1, 4), (1, 4)),
+        ((4, 2), (4, 1)),
+        ((4, 2), (4, 3)),
+        ((4, 4), (4, 3)),
+        ((4, 6), (4, 3)),
+    ]
+    rconds = [None, "random", "warn"]
+    dtypes = ['float32', 'float64']
+    for rcond, hybridize in itertools.product(rconds, [True, False]):
+        for dtype in dtypes:
+            for a_shape, b_shape in shapes:
+                rtol = 1e-2 if dtype == 'float32' else 1e-3
+                atol = 1e-4 if dtype == 'float32' else 1e-5
+                if rcond == "random":
+                    rcond = _np.random.uniform(100, 200)
+                test_lstsq = TestLstsq(rcond)
+                if hybridize:
+                    test_lstsq.hybridize()
+                a_np = _np.random.uniform(-10.0, 10.0, a_shape)
+                b_np = _np.random.uniform(-10.0, 10.0, b_shape)
+                a = np.array(a_np, dtype=dtype)
+                b = np.array(b_np, dtype=dtype)
+                x, residuals, rank, s = test_lstsq(a, b)
+                # check lstsq validity
+                check_lstsq(a_np, b_np, rcond, x, residuals, rank, s)
+
+
 @with_seed()
 @use_np
 def test_np_linalg_pinv():

From 0a1f2968cd71bcf4c0d0a3c62fcf6adcd9b7b858 Mon Sep 17 00:00:00 2001
From: "D. Roberts" <denisa.roberts@denisaroberts.me>
Date: Thu, 9 Apr 2020 17:22:20 -0400
Subject: [PATCH 27/44] Add np.linalg.qr (#17851)

---
 python/mxnet/ndarray/numpy/linalg.py          |  63 ++-
 python/mxnet/numpy/fallback_linalg.py         |   4 +-
 python/mxnet/numpy/linalg.py                  |  61 ++-
 python/mxnet/numpy_dispatch_protocol.py       |   1 +
 python/mxnet/symbol/numpy/linalg.py           |  63 ++-
 src/operator/c_lapack_api.cc                  |  20 +
 src/operator/c_lapack_api.h                   |  72 +++
 src/operator/numpy/linalg/np_qr-inl.h         | 481 ++++++++++++++++++
 src/operator/numpy/linalg/np_qr.cc            | 105 ++++
 src/operator/numpy/linalg/np_qr.cu            |  40 ++
 .../unittest/test_numpy_interoperability.py   |  16 +-
 tests/python/unittest/test_numpy_op.py        |  76 +++
 12 files changed, 985 insertions(+), 17 deletions(-)
 create mode 100644 src/operator/numpy/linalg/np_qr-inl.h
 create mode 100644 src/operator/numpy/linalg/np_qr.cc
 create mode 100644 src/operator/numpy/linalg/np_qr.cu

diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index e153ccb792a5..112a52a6037f 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -22,8 +22,8 @@
 from . import _internal as _npi
 from . import _api_internal
 
-__all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
 
 
 def lstsq(a, b, rcond='warn'):
@@ -469,6 +469,65 @@ def cholesky(a):
     return _api_internal.cholesky(a, True)
 
 
+def qr(a, mode='reduced'):
+    r"""
+    Compute the qr factorization of a matrix a.
+    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.
+
+    Parameters
+    ----------
+    a : (..., M, N) ndarray
+        Matrix or stack of matrices to be qr factored.
+    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional
+        Only default mode, 'reduced', is implemented. If K = min(M, N), then
+        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)
+
+    Returns
+    -------
+    q : (..., M, K) ndarray
+        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).
+    r : (..., K, N) ndarray
+        A matrix or stack of upper triangular matrices.
+
+    Raises
+    ------
+    MXNetError
+        If factoring fails.
+
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> a = np.random.uniform(-10, 10, (2, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.22121978, -0.97522414],
+           [-0.97522414,  0.22121954]])
+    >>> r
+    array([[-4.4131265 , -7.1255064 ],
+           [ 0.        , -0.28771925]])
+    >>> a = np.random.uniform(-10, 10, (2, 3))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.28376842, -0.9588929 ],
+           [-0.9588929 ,  0.28376836]])
+    >>> r
+    array([[-7.242763  , -0.5673361 , -2.624416  ],
+           [ 0.        , -7.297918  , -0.15949416]])
+    >>> a = np.random.uniform(-10, 10, (3, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.34515655,  0.10919492],
+           [ 0.14765628, -0.97452265],
+           [-0.92685735, -0.19591334]])
+    >>> r
+    array([[-8.453794,  8.4175  ],
+           [ 0.      ,  5.430561]])
+    """
+    if mode is not None and mode != 'reduced':
+        raise NotImplementedError("Only default mode='reduced' is implemented.")
+    return tuple(_npi.qr(a))
+
+
 def inv(a):
     r"""
     Compute the (multiplicative) inverse of a matrix.
diff --git a/python/mxnet/numpy/fallback_linalg.py b/python/mxnet/numpy/fallback_linalg.py
index 944a298453ab..5e06ff94a4ce 100644
--- a/python/mxnet/numpy/fallback_linalg.py
+++ b/python/mxnet/numpy/fallback_linalg.py
@@ -25,12 +25,10 @@
     'cond',
     'matrix_power',
     'matrix_rank',
-    'multi_dot',
-    'qr',
+    'multi_dot'
 ]
 
 cond = onp.linalg.cond
 matrix_power = onp.linalg.matrix_power
 matrix_rank = onp.linalg.matrix_rank
 multi_dot = onp.linalg.multi_dot
-qr = onp.linalg.qr
diff --git a/python/mxnet/numpy/linalg.py b/python/mxnet/numpy/linalg.py
index f212c41a7872..445adfdeaeae 100644
--- a/python/mxnet/numpy/linalg.py
+++ b/python/mxnet/numpy/linalg.py
@@ -21,8 +21,8 @@
 from .fallback_linalg import *  # pylint: disable=wildcard-import,unused-wildcard-import
 from . import fallback_linalg
 
-__all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
 __all__ += fallback_linalg.__all__
 
 
@@ -362,6 +362,63 @@ def cholesky(a):
     return _mx_nd_np.linalg.cholesky(a)
 
 
+def qr(a, mode='reduced'):
+    r"""
+    Compute the qr factorization of a matrix a.
+    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.
+
+    Parameters
+    ----------
+    a : (..., M, N) ndarray
+        Matrix or stack of matrices to be qr factored.
+    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional
+        Only default mode, 'reduced', is implemented. If K = min(M, N), then
+        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)
+
+    Returns
+    -------
+    q : (..., M, K) ndarray
+        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).
+    r : (..., K, N) ndarray
+        A matrix or stack of upper triangular matrices.
+
+    Raises
+    ------
+    MXNetError
+        If factoring fails.
+
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> a = np.random.uniform(-10, 10, (2, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.22121978, -0.97522414],
+           [-0.97522414,  0.22121954]])
+    >>> r
+    array([[-4.4131265 , -7.1255064 ],
+           [ 0.        , -0.28771925]])
+    >>> a = np.random.uniform(-10, 10, (2, 3))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.28376842, -0.9588929 ],
+           [-0.9588929 ,  0.28376836]])
+    >>> r
+    array([[-7.242763  , -0.5673361 , -2.624416  ],
+           [ 0.        , -7.297918  , -0.15949416]])
+    >>> a = np.random.uniform(-10, 10, (3, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.34515655,  0.10919492],
+           [ 0.14765628, -0.97452265],
+           [-0.92685735, -0.19591334]])
+    >>> r
+    array([[-8.453794,  8.4175  ],
+           [ 0.      ,  5.430561]])
+    """
+    return _mx_nd_np.linalg.qr(a, mode)
+
+
 def inv(a):
     r"""
     Compute the (multiplicative) inverse of a matrix.
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index 6357fe779dbc..f25bddf8662d 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -164,6 +164,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'linalg.eig',
     'linalg.eigvalsh',
     'linalg.eigh',
+    'linalg.qr',
     'shape',
     'trace',
     'tril',
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index c34cceaffb74..81740dd0ac2e 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -22,8 +22,8 @@
 from . import _op as _mx_sym_np
 from . import _internal as _npi
 
-__all__ = ['norm', 'svd', 'cholesky', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve', 'pinv',
-           'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+__all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
 
 
 def lstsq(a, b, rcond='warn'):
@@ -446,6 +446,65 @@ def cholesky(a):
     return _npi.cholesky(a, True)
 
 
+def qr(a, mode='reduced'):
+    r"""
+    Compute the qr factorization of a matrix a.
+    Factor the matrix a as qr, where q is orthonormal and r is upper-triangular.
+
+    Parameters
+    ----------
+    a : (..., M, N) _Symbol
+        Matrix or stack of matrices to be qr factored.
+    mode: {‘reduced’, ‘complete’, ‘r’, ‘raw’, ‘full’, ‘economic’}, optional
+        Only default mode, 'reduced', is implemented. If K = min(M, N), then
+        * 'reduced’ : returns q, r with dimensions (M, K), (K, N) (default)
+
+    Returns
+    -------
+    q : (..., M, K) _Symbol
+        A matrix or stack of matrices with K orthonormal columns, with K = min(M, N).
+    r : (..., K, N) _Symbol
+        A matrix or stack of upper triangular matrices.
+
+    Raises
+    ------
+    MXNetError
+        If factoring fails.
+
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> a = np.random.uniform(-10, 10, (2, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.22121978, -0.97522414],
+           [-0.97522414,  0.22121954]])
+    >>> r
+    array([[-4.4131265 , -7.1255064 ],
+           [ 0.        , -0.28771925]])
+    >>> a = np.random.uniform(-10, 10, (2, 3))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.28376842, -0.9588929 ],
+           [-0.9588929 ,  0.28376836]])
+    >>> r
+    array([[-7.242763  , -0.5673361 , -2.624416  ],
+           [ 0.        , -7.297918  , -0.15949416]])
+    >>> a = np.random.uniform(-10, 10, (3, 2))
+    >>> q, r = np.linalg.qr(a)
+    >>> q
+    array([[-0.34515655,  0.10919492],
+           [ 0.14765628, -0.97452265],
+           [-0.92685735, -0.19591334]])
+    >>> r
+    array([[-8.453794,  8.4175  ],
+           [ 0.      ,  5.430561]])
+    """
+    if mode is not None and mode != 'reduced':
+        raise NotImplementedError("Only default mode='reduced' is implemented.")
+    return _npi.qr(a)
+
+
 def inv(a):
     r"""
     Compute the (multiplicative) inverse of a matrix.
diff --git a/src/operator/c_lapack_api.cc b/src/operator/c_lapack_api.cc
index 90ce94812a59..13f24c90f6c4 100644
--- a/src/operator/c_lapack_api.cc
+++ b/src/operator/c_lapack_api.cc
@@ -98,6 +98,13 @@
     return 1; \
   }
 
+  #define MXNET_LAPACK_CWRAPPER10(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, dtype* a, \
+                          int lda, dtype* tau, dtype* work, int lwork) { \
+    LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \
+    return 1; \
+  }
+
   #define MXNET_LAPACK_CWRAPPER11(func, dtype) \
   int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int nrhs, \
                           dtype *a, int lda, dtype *b, int ldb, \
@@ -107,6 +114,13 @@
     return 1; \
   }
 
+  #define MXNET_LAPACK_CWRAPPER12(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int k, dtype* a, \
+                          int lda, dtype* tau, dtype* work, int lwork) { \
+    LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \
+    return 1; \
+  }
+
   #define MXNET_LAPACK_UNAVAILABLE(func) \
   int mxnet_lapack_##func(...) { \
     LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \
@@ -146,7 +160,13 @@
   MXNET_LAPACK_CWRAPPER9(sgesdd, float)
   MXNET_LAPACK_CWRAPPER9(dgesdd, double)
 
+  MXNET_LAPACK_CWRAPPER10(sgeqrf, float)
+  MXNET_LAPACK_CWRAPPER10(dgeqrf, double)
+
   MXNET_LAPACK_CWRAPPER11(sgelsd, float)
   MXNET_LAPACK_CWRAPPER11(dgelsd, double)
 
+  MXNET_LAPACK_CWRAPPER12(sorgqr, float)
+  MXNET_LAPACK_CWRAPPER12(dorgqr, double)
+
 #endif  // MSHADOW_USE_MKL == 0
diff --git a/src/operator/c_lapack_api.h b/src/operator/c_lapack_api.h
index d3b844b704ab..f87e040920d8 100644
--- a/src/operator/c_lapack_api.h
+++ b/src/operator/c_lapack_api.h
@@ -290,6 +290,32 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAP_ORGLQ(s, float)
   MXNET_LAPACK_CWRAP_ORGLQ(d, double)
 
+  #define MXNET_LAPACK_CWRAP_GEQRF(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##geqrf(int matrix_layout, int m, int n, \
+                                          dtype *a, int lda, dtype *tau, \
+                                          dtype *work, int lwork) { \
+    if (lwork != -1) { \
+      return LAPACKE_##prefix##geqrf(matrix_layout, m, n, a, lda, tau); \
+    } \
+    *work = 0; \
+    return 0; \
+  }
+  MXNET_LAPACK_CWRAP_GEQRF(s, float)
+  MXNET_LAPACK_CWRAP_GEQRF(d, double)
+
+  #define MXNET_LAPACK_CWRAP_ORGQR(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##orgqr(int matrix_layout, int m, int n, int k, \
+                                          dtype *a, int lda, dtype *tau, \
+                                          dtype *work, int lwork) { \
+    if (lwork != -1) { \
+      return LAPACKE_##prefix##orgqr(matrix_layout, m, n, k, a, lda, tau); \
+    } \
+    *work = 0; \
+    return 0; \
+  }
+  MXNET_LAPACK_CWRAP_ORGQR(s, float)
+  MXNET_LAPACK_CWRAP_ORGQR(d, double)
+
   // This has to be called internally in COL_MAJOR format even when matrix_layout
   // is row-major as otherwise the eigenvectors would be returned as cols in a
   // row-major matrix layout (see MKL documentation).
@@ -480,6 +506,38 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAP_ORGLQ(s, float)
   MXNET_LAPACK_CWRAP_ORGLQ(d, double)
 
+  #define MXNET_LAPACK_CWRAP_GEQRF(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##geqrf(int matrix_layout, int m, int n, \
+                                          dtype *a, int lda, dtype* tau, \
+                                          dtype* work, int lwork) { \
+    if (matrix_layout == MXNET_LAPACK_ROW_MAJOR) { \
+      CHECK(false) << "MXNET_LAPACK_" << #prefix << "geqrf implemented for col-major layout only"; \
+      return 1; \
+    } else { \
+      int info(0); \
+      prefix##geqrf_(&m, &n, a, &lda, tau, work, &lwork, &info); \
+      return info; \
+    } \
+  }
+  MXNET_LAPACK_CWRAP_GEQRF(s, float)
+  MXNET_LAPACK_CWRAP_GEQRF(d, double)
+
+  #define MXNET_LAPACK_CWRAP_ORGQR(prefix, dtype) \
+  inline int MXNET_LAPACK_##prefix##orgqr(int matrix_layout, int m, int n, int k, \
+                                          dtype *a, int lda, dtype* tau, \
+                                          dtype* work, int lwork) { \
+    if (matrix_layout == MXNET_LAPACK_ROW_MAJOR) { \
+      CHECK(false) << "MXNET_LAPACK_" << #prefix << "orgqr implemented for col-major layout only"; \
+      return 1; \
+    } else { \
+      int info(0); \
+      prefix##orgqr_(&m, &n, &k, a, &lda, tau, work, &lwork, &info); \
+      return info; \
+    } \
+  }
+  MXNET_LAPACK_CWRAP_ORGQR(s, float)
+  MXNET_LAPACK_CWRAP_ORGQR(d, double)
+
   // Note: Supports row-major format only. Internally, column-major is used, so all
   // inputs/outputs are flipped (in particular, uplo is flipped).
   #define MXNET_LAPACK_CWRAP_SYEVD(func, dtype) \
@@ -675,12 +733,20 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
                           dtype *vt, int ldvt, \
                           dtype *work, int lwork, int *iwork);
 
+  #define MXNET_LAPACK_CWRAPPER10(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, dtype* a, \
+                          int lda, dtype* tau, dtype* work, int lwork);
+
   #define MXNET_LAPACK_CWRAPPER11(func, dtype) \
   int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int nrhs, \
                           dtype *a, int lda, dtype *b, int ldb, \
                           dtype *s, dtype rcond, int *rank, \
                           dtype *work, int lwork, int *iwork);
 
+  #define MXNET_LAPACK_CWRAPPER12(func, dtype) \
+  int MXNET_LAPACK_##func(int matrix_layout, int m, int n, int k, dtype* a, \
+                          int lda, dtype* tau, dtype* work, int lwork);
+
   #define MXNET_LAPACK_UNAVAILABLE(func) \
   int mxnet_lapack_##func(...);
   MXNET_LAPACK_CWRAPPER1(spotrf, float)
@@ -717,9 +783,15 @@ inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda) {
   MXNET_LAPACK_CWRAPPER9(sgesdd, float)
   MXNET_LAPACK_CWRAPPER9(dgesdd, double)
 
+  MXNET_LAPACK_CWRAPPER10(sgeqrf, float)
+  MXNET_LAPACK_CWRAPPER10(dgeqrf, double)
+
   MXNET_LAPACK_CWRAPPER11(sgelsd, float)
   MXNET_LAPACK_CWRAPPER11(dgelsd, double)
 
+  MXNET_LAPACK_CWRAPPER12(sorgqr, float)
+  MXNET_LAPACK_CWRAPPER12(dorgqr, double)
+
   #undef MXNET_LAPACK_CWRAPPER1
   #undef MXNET_LAPACK_CWRAPPER2
   #undef MXNET_LAPACK_CWRAPPER3
diff --git a/src/operator/numpy/linalg/np_qr-inl.h b/src/operator/numpy/linalg/np_qr-inl.h
new file mode 100644
index 000000000000..c33e12059598
--- /dev/null
+++ b/src/operator/numpy/linalg/np_qr-inl.h
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_qr-inl.h
+ * \brief Function definition of the QR Operator.
+ */
+#ifndef MXNET_OPERATOR_NUMPY_LINALG_NP_QR_INL_H_
+#define MXNET_OPERATOR_NUMPY_LINALG_NP_QR_INL_H_
+
+#include <mxnet/operator_util.h>
+#include <vector>
+#include <algorithm>
+#include "../../tensor/la_op.h"
+#include "../../tensor/la_op-inl.h"
+#include "../../linalg.h"
+#include "../../operator_common.h"
+#include "../../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+using namespace mshadow;
+
+//////////////////////////////// QR ////////////////////////////////////////////
+
+// CPU/GPU-versions of LAPACK function "geqrf", "orgqr". Please refer to the
+// LAPACK documentation for further details.
+// Note:
+// - Both functions have A as input and output parameter
+// - Both functions require extra workspace, passed as 1D tensor
+// - We call orgqr after geqrf. Apart from A, they also communicate via the
+//   first part of the workspace.
+
+template<typename xpu, typename DType>
+void linalg_geqrf(const Tensor<xpu, 2, DType>& A,
+                  const Tensor<xpu, 1, DType>& work,
+                  Stream<xpu> *s = 0);
+
+template<typename xpu, typename DType>
+void linalg_orgqr(const Tensor<xpu, 2, DType>& A,
+                  const Tensor<xpu, 1, DType>& work,
+                  Stream<xpu> *s = 0);
+
+// This function determines the amount of workspace needed for linalg_geqrf,
+// linalg_orgqr. The workspace can be used for both. The first mn entries are
+// used to communicate information from geqrf to orgqr (tau).
+
+template<typename xpu, typename DType>
+int linalg_qr_workspace_query(const Tensor<xpu, 2, DType>& A,
+                              Stream<xpu> *s = 0);
+
+//////////////////////////////// QR ////////////////////////////////////////////
+
+// CPU/GPU-versions of LAPACK function "geqrf", "orqrq." to be used in QR op.
+
+template<typename xpu, typename DType> inline
+void check_geqrf(const Tensor<xpu, 2, DType>& A,
+                 const Tensor<xpu, 1, DType>& work) {
+  CHECK_LE(A.size(0), work.size(0))
+    << "Size of work is too small";
+}
+
+// transpose helper
+struct QrTypeTransposeHelper {
+  template<typename InDType, typename OutDType>
+  MSHADOW_XINLINE static void Map(int i, const InDType *in_data, OutDType *out_data,
+                                  const int ncol1, const int ncol2, const int step) {
+    int idx = i / step, row = (i % step) / ncol1, col = (i % step) % ncol1;
+    out_data[idx * step + row + col * ncol2] = static_cast<OutDType>(in_data[i]);
+  }
+};
+
+#define LINALG_CPU_GEQRF(fname, DType) \
+template<> inline \
+void linalg_geqrf<cpu, DType>(const Tensor<cpu, 2, DType>& A, \
+                              const Tensor<cpu, 1, DType>& work, \
+                              Stream<cpu> *s) { \
+  check_geqrf(A, work); \
+  const int m = A.size(1); \
+  const int n = A.size(0); \
+  const int mn = (m > n ? n : m); \
+  int lwork = work.shape_.Size() - mn; \
+  int ret(MXNET_LAPACK_##fname(MXNET_LAPACK_COL_MAJOR, m, n, \
+                               A.dptr_, A.stride_, work.dptr_, \
+                               work.dptr_ + mn, lwork)); \
+  CHECK_EQ(ret, 0) << #fname << " failed in lapack on cpu."; \
+}
+LINALG_CPU_GEQRF(sgeqrf, float)
+LINALG_CPU_GEQRF(dgeqrf, double)
+
+#define LINALG_CPU_ORGQR(fname, DType) \
+template<> inline \
+void linalg_orgqr<cpu, DType>(const Tensor<cpu, 2, DType>& A, \
+                              const Tensor<cpu, 1, DType>& work, \
+                              Stream<cpu> *s) { \
+  check_geqrf(A, work); \
+  const int m = A.size(1); \
+  const int n = A.size(0); \
+  const int mn = (m > n ? n : m); \
+  int lwork = work.shape_.Size() - mn; \
+  int ret(MXNET_LAPACK_##fname(MXNET_LAPACK_COL_MAJOR, m, mn, mn, \
+                               A.dptr_, A.stride_, work.dptr_, \
+                               work.dptr_ + mn, lwork)); \
+  CHECK_EQ(ret, 0) << #fname << " failed in lapack on cpu."; \
+}
+LINALG_CPU_ORGQR(sorgqr, float)
+LINALG_CPU_ORGQR(dorgqr, double)
+
+#define LINALG_CPU_QR_WORKSPACE_QUERY(prefix, DType) \
+template<> inline \
+int linalg_qr_workspace_query<cpu, DType>(const Tensor<cpu, 2, DType>& A, \
+                                          Stream<cpu> *s) { \
+  const int m(A.size(1)); \
+  const int n(A.size(0)); \
+  const int mn = (m > n ? n : m); \
+  DType work = 0; \
+  int ret(MXNET_LAPACK_##prefix##geqrf(MXNET_LAPACK_COL_MAJOR, m, \
+                                       n, A.dptr_, A.stride_, &work, \
+                                       &work, -1)); \
+  CHECK_EQ(ret, 0) << #prefix << "geqrf: Workspace query failed on CPU."; \
+  int ws_size(static_cast<int>(work)); \
+  ret = MXNET_LAPACK_##prefix##orgqr(MXNET_LAPACK_COL_MAJOR, m, mn, \
+                                     mn, A.dptr_, \
+                                     A.stride_, &work, &work, -1); \
+  CHECK_EQ(ret, 0) << #prefix << "orgqr: Workspace query failed on CPU."; \
+  int wsz2(static_cast<int>(work)); \
+  if (wsz2 > ws_size) ws_size = wsz2; \
+  return ws_size + mn; \
+}
+LINALG_CPU_QR_WORKSPACE_QUERY(s, float)
+LINALG_CPU_QR_WORKSPACE_QUERY(d, double)
+
+#ifdef __CUDACC__
+
+#define LINALG_GPU_GEQRF(fname, DType) \
+template<> inline \
+void linalg_geqrf<gpu, DType>(const Tensor<gpu, 2, DType>& A, \
+                              const Tensor<gpu, 1, DType>& work, \
+                              Stream<gpu> *s) { \
+  using namespace mxnet; \
+  using mshadow::gpu; \
+  CHECK_NOTNULL(s); \
+  check_geqrf(A, work); \
+  const int m = A.size(1); \
+  const int n = A.size(0); \
+  const int mn = (m > n ? n : m); \
+  int lwork(work.size(0) - mn); \
+  Storage::Handle info = Storage::Get()->Alloc(sizeof(int), Context::GPU()); \
+  CUSOLVER_CALL(cusolver##fname(Stream<gpu>::GetSolverHandle(s), \
+                m, n, A.dptr_, A.stride_, work.dptr_, \
+                work.dptr_ + mn, lwork, static_cast<int *>(info.dptr))); \
+  Storage::Get()->Free(info); \
+}
+LINALG_GPU_GEQRF(DnSgeqrf, float)
+LINALG_GPU_GEQRF(DnDgeqrf, double)
+
+// ORGQR only available with cuda8 or higher.
+#if CUDA_VERSION >= 8000
+
+#define LINALG_GPU_ORGQR(fname, DType) \
+template<> inline \
+void linalg_orgqr<gpu, DType>(const Tensor<gpu, 2, DType>& A, \
+                              const Tensor<gpu, 1, DType>& work, \
+                              Stream<gpu> *s) { \
+  using namespace mxnet; \
+  using mshadow::gpu; \
+  CHECK_NOTNULL(s); \
+  check_geqrf(A, work); \
+  const int m = A.size(1); \
+  const int n = A.size(0); \
+  const int mn = (m > n ? n : m); \
+  int lwork(work.size(0) - mn); \
+  Storage::Handle info = Storage::Get()->Alloc(sizeof(int), Context::GPU()); \
+  CUSOLVER_CALL(cusolver##fname(Stream<gpu>::GetSolverHandle(s), \
+                m, mn, mn, A.dptr_, A.stride_, work.dptr_, \
+                work.dptr_ + mn, lwork, static_cast<int *>(info.dptr))); \
+  Storage::Get()->Free(info); \
+}
+
+#else
+
+#define LINALG_GPU_ORGQR(fname, DType) \
+template<> inline \
+void linalg_orgqr<gpu, DType>(const Tensor<gpu, 2, DType>& A, \
+                              const Tensor<gpu, 1, DType>& work, \
+                              Stream<gpu> *s) { \
+  LOG(FATAL) << "orgqr requires CUDA version >= 8.0!"; \
+}
+
+#endif  // CUDA_VERSION >= 8000
+
+LINALG_GPU_ORGQR(DnSorgqr, float)
+LINALG_GPU_ORGQR(DnDorgqr, double)
+
+// ORGQR only available with cuda8 or higher.
+#if CUDA_VERSION >= 8000
+
+#define LINALG_GPU_QR_WORKSPACE_QUERY(prefix, DType) \
+template<> inline \
+int linalg_qr_workspace_query<gpu, DType>(const Tensor<gpu, 2, DType>& A, \
+                                          Stream<gpu> *s) { \
+  using namespace mxnet; \
+  using mshadow::gpu; \
+  const int m(A.size(1)); \
+  const int n(A.size(0)); \
+  const int mn = (m > n ? n : m); \
+  int work1(0); \
+  CUSOLVER_CALL(cusolverDn##prefix##geqrf_bufferSize(Stream<gpu>::GetSolverHandle(s), \
+                m, n, A.dptr_, A.stride_, &work1)); \
+  int work2(0);  \
+  Storage::Handle tau = Storage::Get()->Alloc(sizeof(DType), Context::GPU()); \
+  CUSOLVER_CALL(cusolverDn##prefix##orgqr_bufferSize(Stream<gpu>::GetSolverHandle(s), \
+                m, mn, mn, A.dptr_, A.stride_, static_cast<DType *>(tau.dptr), &work2)); \
+  Storage::Get()->Free(tau); \
+  return std::max(work1, work2) + mn; \
+}
+
+#else
+
+#define LINALG_GPU_QR_WORKSPACE_QUERY(prefix, DType) \
+template<> inline \
+int linalg_qr_workspace_query<gpu, DType>(const Tensor<gpu, 2, DType>& A, \
+                                          Stream<gpu> *s) { \
+  LOG(FATAL) << "orgqr requires CUDA version >= 8.0!"; \
+  return 0; \
+}
+
+#endif  // CUDA_VERSION >= 8000
+
+LINALG_GPU_QR_WORKSPACE_QUERY(S, float)
+LINALG_GPU_QR_WORKSPACE_QUERY(D, double)
+
+#endif  // __CUDACC__
+
+// (Q, R) = qr(A)
+// - Works on transposed A, At of shape (n, m) = Qt of shape (m, k) @ Rt of shape (k, n)
+// - k = min(m, n)
+// - Needs workspace (DType), size of which is determined by a workspace query
+
+struct qr {
+  template<typename xpu, typename DType>
+  static void op(const Tensor<xpu, 3, DType>& a,
+                 const Tensor<xpu, 3, DType>& q,
+                 const Tensor<xpu, 1, DType>& work,
+                 const OpContext& ctx,
+                 const nnvm::NodeAttrs& attrs) {
+    Stream<xpu> *s = ctx.get_stream<xpu>();
+    const mxnet::TShape& a_shape = a.shape_;
+    for (index_t i = 0; i < a_shape[0]; ++i) {
+      const Tensor<xpu, 2, DType>& ai = a[i];
+      const Tensor<xpu, 2, DType>& qi = q[i];
+      linalg_geqrf(ai, work, s);  // get R
+      Copy(qi, ai, s);
+      linalg_orgqr(qi, work, s);  // get Q
+    }
+  }
+};
+
+// Calculate the necessary workspace size
+template<typename xpu>
+size_t QrForwardWorkspaceSize(const TBlob& a,
+                              const TBlob& q,
+                              const TBlob& r,
+                              const std::vector<OpReqType>& req,
+                              const OpContext& ctx) {
+  if (kNullOp == req[0]) { return 0U; }
+  // Zero-size input, no need to launch kernel
+  if (0U == a.Size()) { return 0U; }
+
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const mxnet::TShape& a_shape = a.shape_;
+  const int a_ndim = a_shape.ndim();
+  const int n = a.size(a_ndim - 1);
+  const int m = a.size(a_ndim - 2);
+
+  MSHADOW_TYPE_SWITCH(a.type_flag_, AType, {
+    MSHADOW_SGL_DBL_TYPE_SWITCH(q.type_flag_, DType, {
+      size_t work_space_size = 0;
+      if (m == n) {
+        // For transposed input matrix a and q
+        work_space_size += 2 * a.Size();
+      } else if (m > n) {
+        // For transposed input matrix a and q of same shape and r transpose
+        work_space_size += 2 * a.Size();
+        work_space_size += r.Size();
+      } else {
+        // For transposed input matrix a and q of same shape and q transpose
+        work_space_size += 2 * a.Size();
+        work_space_size += q.Size();
+      }
+      // For workspace size in query; done for all (m, n) shapes
+      Tensor<xpu, 2, AType> a_temp_tensor = a.FlatToKD<xpu, 3, AType>(s)[0];
+      if (xpu::kDevCPU) {
+        std::vector<DType> A_data(a_temp_tensor.MSize(), 0);
+        TBlob a_data(A_data.data(), a_temp_tensor.shape_, a.dev_mask(), a.dev_id());
+        work_space_size +=
+          linalg_qr_workspace_query(a_data.get<xpu, 2, DType>(s), s);
+      } else {
+        Storage::Handle a_handle =
+          Storage::Get()->Alloc(sizeof(DType) * a_temp_tensor.shape_.Size(), Context::GPU());
+        TBlob a_data(static_cast<DType*>(a_handle.dptr), a_temp_tensor.shape_,
+                                         a.dev_mask(), a.dev_id());
+        work_space_size +=
+          linalg_qr_workspace_query(a_data.get<xpu, 2, DType>(s), s);
+        Storage::Get()->Free(a_handle);
+      }
+      return work_space_size * sizeof(DType);
+    });
+  });
+  LOG(FATAL) << "InternalError: cannot reach here";
+  return 0U;
+}
+
+template<typename xpu>
+void QrOpForwardImpl(const TBlob& a,
+                     const TBlob& q,
+                     const TBlob& r,
+                     const std::vector<OpReqType>& req,
+                     const Tensor<xpu, 1, char>& workspace,
+                     const OpContext& ctx,
+                     const nnvm::NodeAttrs& attrs) {
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const mxnet::TShape& a_shape = a.shape_;
+  const int a_ndim = a_shape.ndim();
+
+  MSHADOW_SGL_DBL_TYPE_SWITCH(q.type_flag_, DType, {
+    const int n = a.size(a_ndim - 1);
+    const int m = a.size(a_ndim - 2);
+    // a shape transposed
+    mxnet::TShape transp_shape(a_shape);
+    transp_shape[a_ndim - 1] = m;
+    transp_shape[a_ndim - 2] = n;
+    // Common for all (m, n) shapes
+    const size_t workspace_size = (workspace.shape_.Size() + sizeof(DType) - 1) / sizeof(DType);
+    DType *a_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+    DType *qt_ptr = a_ptr + a_shape.Size();
+    TBlob a_trans_data(a_ptr, transp_shape, a.dev_mask(), a.dev_id());
+    TBlob q_trans_data(qt_ptr, transp_shape, a.dev_mask(), a.dev_id());
+    Tensor<xpu, 3, DType> At = a_trans_data.FlatToKD<xpu, 3, DType>(s);
+    Tensor<xpu, 3, DType> Qt = q_trans_data.FlatToKD<xpu, 3, DType>(s);
+    Tensor<xpu, 3, DType> R = r.FlatToKD<xpu, 3, DType>(s);
+    Tensor<xpu, 3, DType> Q = q.FlatToKD<xpu, 3, DType>(s);
+
+    MSHADOW_TYPE_SWITCH(a.type_flag_, AType, {
+      // Cast type and transpose a
+        mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+          s, a.Size(), a.dptr<AType>(), a_ptr, n, m, m * n);
+      });
+
+    if (m == n) {
+      DType *work_ptr = qt_ptr + a_shape.Size();
+      TBlob work_data(work_ptr, Shape1(workspace_size - a_shape.Size()),
+                      a.dev_mask(), a.dev_id());
+      Tensor<xpu, 1, DType> Work = work_data.get<xpu, 1, DType>(s);
+      qr::op(At,
+             Qt,
+             Work, ctx, attrs);
+      // Transpose into R
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, r.Size(), At.dptr_, R.dptr_, m, m, m * m);
+      // Transpose into Q
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, q.Size(), Qt.dptr_, Q.dptr_, m, m, m * m);
+      // R is triu
+      mxnet_op::Kernel<ZeroTriangular, xpu>::Launch(
+        s, R.MSize(), m * R.stride_, R.stride_, R.dptr_, true);
+    } else if (m > n) {
+      // r shape transposed
+      mxnet::TShape r_shape(transp_shape);
+      r_shape[a_ndim - 1] = n;
+      DType *rtemp_ptr = qt_ptr + a_shape.Size();
+      DType *work_ptr = rtemp_ptr + r_shape.Size();
+      TBlob rtemp_data(rtemp_ptr, r_shape, a.dev_mask(), a.dev_id());
+      TBlob work_data(work_ptr, Shape1(workspace_size - 2 * a_shape.Size()),
+                      a.dev_mask(), a.dev_id());
+      Tensor<xpu, 3, DType> Rtemp = rtemp_data.FlatToKD<xpu, 3, DType>(s);
+      Tensor<xpu, 1, DType> Work = work_data.get<xpu, 1, DType>(s);
+      qr::op(At,
+             Qt,
+             Work, ctx, attrs);
+      // Final Rt of shape (N, N)
+      for (index_t i = 0; i < At.size(0); ++i) {
+        const Tensor<xpu, 2, DType>& Ati = At[i];
+        const Tensor<xpu, 2, DType>& Rtempi = Rtemp[i];
+        Tensor<xpu, 2, DType> Rk(Ati.dptr_, Shape2(n, n), Ati.stride_, s);
+        Copy(Rtempi, Rk, s);
+      }
+      // Transpose into R of shape (N, N)
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, r.Size(), Rtemp.dptr_, R.dptr_, n, n, n * n);
+      // Transpose resulting qt(N, M) into q of shape (M, N)
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, q.Size(), Qt.dptr_, Q.dptr_, m, n, m * n);
+      // R is triu
+      mxnet_op::Kernel<ZeroTriangular, xpu>::Launch(
+        s, R.MSize(), n * R.stride_, R.stride_, R.dptr_, true);
+    // case m < n
+    } else {
+      // q shape transposed
+      mxnet::TShape q_shape(transp_shape);
+      q_shape[a_ndim - 2] = m;
+      DType *qtemp_ptr = qt_ptr + a_shape.Size();
+      DType *work_ptr = qtemp_ptr + q_shape.Size();
+      TBlob qtemp_data(qtemp_ptr, q_shape, a.dev_mask(), a.dev_id());
+      TBlob work_data(work_ptr, Shape1(workspace_size - 2 * a_shape.Size()),
+                      a.dev_mask(), a.dev_id());
+      Tensor<xpu, 3, DType> Qtemp = qtemp_data.FlatToKD<xpu, 3, DType>(s);
+      Tensor<xpu, 1, DType> Work = work_data.get<xpu, 1, DType>(s);
+      qr::op(At,
+             Qt,
+             Work, ctx, attrs);
+      // Transpose into R of shape (M, N)
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, r.Size(), At.dptr_, R.dptr_, m, n, n * m);
+      // Get Qt(M, M) from Qt(N, M)
+      for (index_t i = 0; i < Qt.size(0); ++i) {
+        const Tensor<xpu, 2, DType>& Qti = Qt[i];
+        const Tensor<xpu, 2, DType>& Qtempi = Qtemp[i];
+        Tensor<xpu, 2, DType> Qk(Qti.dptr_, Shape2(m, m), Qti.stride_, s);
+        Copy(Qtempi, Qk, s);
+      }
+      // Transpose resulting qt into q of shape (M, M)
+      mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+        s, q.Size(), Qtemp.dptr_, Q.dptr_, m, m, m * m);
+      // R is triu
+      mxnet_op::Kernel<ZeroTriangular, xpu>::Launch(
+        s, R.MSize(), m * R.stride_, R.stride_, R.dptr_, true);
+    }
+  });
+}
+
+// (A) => (Q, R)
+template<typename xpu>
+void NumpyLaQrForward(const nnvm::NodeAttrs& attrs,
+                      const OpContext& ctx,
+                      const std::vector<TBlob>& inputs,
+                      const std::vector<OpReqType>& req,
+                      const std::vector<TBlob>& outputs) {
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 2U);
+  CHECK_EQ(req.size(), 2U);
+
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const TBlob& a = inputs[0];
+  const TBlob& q = outputs[0];
+  const TBlob& r = outputs[1];
+
+  if (kNullOp == req[0]) { return; }
+  // Zero-size input, no need to launch kernel
+  if (0U == a.Size()) { return; }
+
+  // Calculate workspace size
+  size_t workspace_size = QrForwardWorkspaceSize<xpu>(a, q, r, req, ctx);
+  Tensor<xpu, 1, char> workspace =
+    ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+  // Op
+  QrOpForwardImpl<xpu>(a, q, r, req, workspace, ctx, attrs);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_LINALG_NP_QR_INL_H_
diff --git a/src/operator/numpy/linalg/np_qr.cc b/src/operator/numpy/linalg/np_qr.cc
new file mode 100644
index 000000000000..c4ef7f3ec783
--- /dev/null
+++ b/src/operator/numpy/linalg/np_qr.cc
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_qr.cc
+ * \brief CPU implementation of the QR Operator
+ */
+#include <mxnet/operator_util.h>
+#include <vector>
+#include "../../mshadow_op.h"
+#include "../../mxnet_op.h"
+#include "../../operator_common.h"
+#include "../../elemwise_op_common.h"
+#include "./np_qr-inl.h"
+
+namespace mxnet {
+namespace op {
+
+// Shape inference function for qr
+// Inputs: A. Outputs: Q, R
+inline bool NumpyLaQrShape(const nnvm::NodeAttrs& attrs,
+                           mxnet::ShapeVector* in_attrs,
+                           mxnet::ShapeVector* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 2U);
+  const mxnet::TShape& in_a = (*in_attrs)[0];
+
+  if (in_a.ndim() >= 2) {
+    // Forward shape inference.
+    const int ndim(in_a.ndim());
+    const int k = in_a[ndim - 2] > in_a[ndim - 1] ? in_a[ndim - 1] : in_a[ndim - 2];
+    // Q
+    std::vector<int> oshape_q(ndim);
+    for (int i = 0; i < ndim - 1; ++i) {
+      oshape_q[i] = in_a[i];
+    }
+    oshape_q[ndim - 1] = k;
+    mxnet::TShape tshape_q(oshape_q.begin(), oshape_q.end());
+    SHAPE_ASSIGN_CHECK(*out_attrs, 0, tshape_q);
+    // R
+    std::vector<int> oshape_r(ndim);
+    for (int i = 0; i < ndim - 2; ++i) {
+      oshape_r[i] = in_a[i];
+    }
+    oshape_r[ndim - 2] = k;
+    oshape_r[ndim - 1] = in_a[ndim - 1];
+    mxnet::TShape tshape_r(oshape_r.begin(), oshape_r.end());
+    SHAPE_ASSIGN_CHECK(*out_attrs, 1, tshape_r);
+    return true;
+  }
+  return false;
+}
+
+inline bool NumpyLaQrType(const nnvm::NodeAttrs& attrs,
+                        std::vector<int>* in_attrs,
+                        std::vector<int>* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 2U);
+  int a_type = in_attrs->at(0);
+  // unsupport float16
+  CHECK_NE(a_type, mshadow::kFloat16)
+    << "array type float16 is unsupported in linalg";
+  if (mshadow::kFloat32 == a_type) {
+    TYPE_ASSIGN_CHECK(*out_attrs, 0, in_attrs->at(0));
+    TYPE_ASSIGN_CHECK(*out_attrs, 1, in_attrs->at(0));
+  } else {
+    TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kFloat64);
+    TYPE_ASSIGN_CHECK(*out_attrs, 1, mshadow::kFloat64);
+  }
+  return out_attrs->at(0) != -1 && out_attrs->at(1) != -1;
+}
+
+NNVM_REGISTER_OP(_npi_qr)
+.describe(R"code()code" ADD_FILELINE)
+.set_num_inputs(1)
+.set_num_outputs(2)
+.set_attr<nnvm::FListInputNames>("FListInputNames", [](const NodeAttrs& attrs) {
+  return std::vector<std::string>{"A"}; })
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyLaQrShape)
+.set_attr<nnvm::FInferType>("FInferType", NumpyLaQrType)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs) {
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace}; })
+.set_attr<FCompute>("FCompute<cpu>", NumpyLaQrForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("A", "NDArray-or-Symbol", "Input matrices to be factorized");
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_qr.cu b/src/operator/numpy/linalg/np_qr.cu
new file mode 100644
index 000000000000..5547d2627381
--- /dev/null
+++ b/src/operator/numpy/linalg/np_qr.cu
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_qr.cu
+ * \brief GPU implementation of the QR Operator
+ */
+
+#include <mxnet/operator_util.h>
+#include <vector>
+#include "./np_qr-inl.h"
+
+namespace mxnet {
+namespace op {
+
+#if MXNET_USE_CUSOLVER == 1
+
+NNVM_REGISTER_OP(_npi_qr)
+.set_attr<FCompute>("FCompute<gpu>", NumpyLaQrForward<gpu>);
+
+#endif
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index c4e4b599e2d6..3c0f01ee938e 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -489,6 +489,13 @@ def _add_workload_linalg_cholesky():
         OpArgMngr.add_workload('linalg.cholesky', np.array(a, dtype=dtype))
 
 
+def _add_workload_linalg_qr():
+    A = np.array([[0, 1], [1, 1], [1, 1], [2, 1]])
+    OpArgMngr.add_workload('linalg.qr', A)
+    # default mode in numpy is 'reduced'
+    OpArgMngr.add_workload('linalg.qr', A, mode='reduced')
+
+
 def _add_workload_linalg_inv():
     OpArgMngr.add_workload('linalg.inv', np.array(_np.ones((0, 0)), dtype=np.float32))
     OpArgMngr.add_workload('linalg.inv', np.array(_np.ones((0, 1, 1)), dtype=np.float64))
@@ -2053,13 +2060,6 @@ def _add_workload_linalg_multi_dot():
     OpArgMngr.add_workload('linalg.multi_dot', [F,F])
 
 
-
-def _add_workload_linalg_qr():
-    A = np.array([[0, 1], [1, 1], [1, 1], [2, 1]])
-    OpArgMngr.add_workload('linalg.qr', A)
-    OpArgMngr.add_workload('linalg.qr', A, mode='r')
-
-
 def _add_workload_heaviside():
     x = np.array([[-30.0, -0.1, 0.0, 0.2], [7.5, np.nan, np.inf, -np.inf]], dtype=np.float64)
     OpArgMngr.add_workload('heaviside', x, 0.5)
@@ -2813,6 +2813,7 @@ def _prepare_workloads():
     _add_workload_zeros_like(array_pool)
     _add_workload_linalg_norm()
     _add_workload_linalg_cholesky()
+    _add_workload_linalg_qr()
     _add_workload_linalg_inv()
     _add_workload_linalg_solve()
     _add_workload_linalg_det()
@@ -2829,7 +2830,6 @@ def _prepare_workloads():
     _add_workload_linalg_matrix_power()
     _add_workload_linalg_matrix_rank()
     _add_workload_linalg_multi_dot()
-    _add_workload_linalg_qr()
     _add_workload_trace()
     _add_workload_tril()
     _add_workload_outer()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 3242a147eff6..4b21853c3433 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -4966,6 +4966,82 @@ def check_svd(UT, L, V, data_np):
                 check_svd(UT, L, V, data_np)
 
 
+@with_seed()
+@use_np
+def test_np_linalg_qr():
+    class TestQR(HybridBlock):
+        def __init__(self):
+            super(TestQR, self).__init__()
+
+        def hybrid_forward(self, F, data):
+            return F.np.linalg.qr(data)
+
+    def check_qr(q, r, a_np):
+        # check Q@R = A
+        t = _np.matmul(q, r)
+        assert t.shape == data_np.shape
+        assert_almost_equal(t, data_np, rtol=rtol, atol=atol)
+        # check QT@Q = I
+        qT = _np.swapaxes(q, -2, -1)
+        I = _np.matmul(qT, q)
+        Ip = _np.eye(I.shape[-2])
+        assert_almost_equal(I, Ip, atol=atol, rtol=rtol)
+        # check original numpy
+        try:
+            q_expected, r_expected = _np.linalg.qr(a_np)
+        except Exception as e:
+            print("a_np", a_np)
+            print("a shape:", a_np.shape)
+            print(e)
+        else:
+            assert q.shape == q_expected.shape
+            assert r.shape == r_expected.shape
+            assert_almost_equal(q.asnumpy(), q_expected, rtol=rtol, atol=atol)
+            assert_almost_equal(r.asnumpy(), r_expected, rtol=rtol, atol=atol)
+    shapes = [
+        (0, 0),
+        (0, 1),
+        (1, 1),
+        (5, 3),
+        (3, 5),
+        (3, 3),
+        (5, 5),
+        (8, 8),
+        (4, 5),
+        (4, 6),
+        (5, 4),
+        (6, 5),
+        (6, 5, 6),
+        (6, 6, 5),
+        (3, 0, 0),
+        (2, 3, 3, 4),
+        (0, 5, 3, 3),
+        (5, 0, 3, 3),
+        (3, 3, 0, 0),
+        (4, 2, 2, 1),
+        (2, 3, 4, 3)
+    ]
+    dtypes = ['float32', 'float64']
+    for hybridize, shape, dtype in itertools.product([True, False], shapes, dtypes):
+        rtol = atol = 0.01
+        test_qr = TestQR()
+        if hybridize:
+            test_qr.hybridize()
+        data_np = _np.random.uniform(-10.0, 10.0, shape)
+        data_np = _np.array(data_np, dtype=dtype)
+        data = np.array(data_np, dtype=dtype)
+
+        ret = test_qr(data)
+        Q, R = ret[0], ret[1]
+        check_qr(Q, R, data_np)
+
+        # check imperative once more; mode='reduced' is default
+        # behavior and optional parameter in original numpy
+        ret = np.linalg.qr(data, mode='reduced')
+        Q, R = ret[0], ret[1]
+        check_qr(Q, R, data_np)
+
+
 @with_seed()
 @use_np
 def test_np_linalg_cholesky():

From ef1400050ce1ed86d0cbed578438591652328da5 Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Tue, 14 Apr 2020 12:58:11 +0800
Subject: [PATCH 28/44] [Numpy] FFI for linalg.qr and linalg.lstsq (#18040)

* impl - ffi for linalg.qr/lstsq

* impl - ffi benchmark

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py     |  2 +
 python/mxnet/ndarray/numpy/linalg.py      | 12 ++---
 python/mxnet/symbol/numpy/linalg.py       |  6 ++-
 src/api/operator/numpy/linalg/np_lstsq.cc | 65 +++++++++++++++++++++++
 src/api/operator/numpy/linalg/np_qr.cc    | 44 +++++++++++++++
 src/operator/numpy/linalg/np_lstsq-inl.h  | 27 ++++++++--
 6 files changed, 143 insertions(+), 13 deletions(-)
 create mode 100644 src/api/operator/numpy/linalg/np_lstsq.cc
 create mode 100644 src/api/operator/numpy/linalg/np_qr.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 09fec7b6d7a5..d484f77ac5cb 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -62,6 +62,8 @@ def prepare_workloads():
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("linalg.cholesky", pool['1x1'])
+    OpArgMngr.add_workload("linalg.qr", pool['3x3'])
+    OpArgMngr.add_workload("linalg.lstsq", pool['2x1'], pool['2'], rcond=None)
     OpArgMngr.add_workload("linalg.eigvals", pool['1x1'])
     OpArgMngr.add_workload("linalg.eigvalsh", pool['1x1'], UPLO='L')
     OpArgMngr.add_workload("linalg.inv", pool['1x1'])
diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index 112a52a6037f..54a01b60bd55 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -94,13 +94,9 @@ def lstsq(a, b, rcond='warn'):
     >>> m, c
     (1.0 -0.95) # may vary
     """
-    new_default = False
-    if rcond is None:
-        rcond = _np.finfo(a.dtype).eps
-        new_default = True
-    if rcond == "warn":
-        rcond = -1
-    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, new_default=new_default)
+    finfo_eps_32 = _np.finfo(_np.float32).eps
+    finfo_eps_64 = _np.finfo(_np.float64).eps
+    x, residuals, rank, s = _api_internal.lstsq(a, b, rcond, finfo_eps_32, finfo_eps_64)
     return (x, residuals, rank, s)
 
 
@@ -525,7 +521,7 @@ def qr(a, mode='reduced'):
     """
     if mode is not None and mode != 'reduced':
         raise NotImplementedError("Only default mode='reduced' is implemented.")
-    return tuple(_npi.qr(a))
+    return tuple(_api_internal.qr(a))
 
 
 def inv(a):
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index 81740dd0ac2e..c16aef00e48d 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -81,12 +81,14 @@ def lstsq(a, b, rcond='warn'):
     If `b` is a matrix, then all array results are returned as matrices.
     """
     new_default = False
+    finfo_eps_32 = _np.finfo(_np.float32).eps
+    finfo_eps_64 = _np.finfo(_np.float64).eps
     if rcond is None:
-        rcond = _np.finfo(_np.float64).eps
+        rcond = 1
         new_default = True
     if rcond == "warn":
         rcond = -1
-    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, new_default=new_default)
+    x, residuals, rank, s = _npi.lstsq(a, b, rcond=rcond, finfoEps32=finfo_eps_32, finfoEps64=finfo_eps_64, new_default=new_default)  # pylint: disable=line-too-long
     return (x, residuals, rank, s)
 
 
diff --git a/src/api/operator/numpy/linalg/np_lstsq.cc b/src/api/operator/numpy/linalg/np_lstsq.cc
new file mode 100644
index 000000000000..fbeafbee6054
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_lstsq.cc
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_lstsq.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_lstsq.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_lstsq-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.lstsq")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_lstsq");
+  nnvm::NodeAttrs attrs;
+  op::LstsqParam param;
+  if (args[2].type_code() == kNull) {
+    param.rcond = static_cast<double>(1);
+  } else if (args[2].type_code() == kStr) {
+    const std::string rcond_str = args[2].operator std::string();
+    if (rcond_str == "warn") {
+      param.rcond = static_cast<double>(-1);
+    } else {
+      CHECK(false) << "ValueError: wrong parameter rcond = " << rcond_str;
+    }
+  } else {
+    param.rcond = args[2].operator double();
+  }
+  param.finfoEps32 = args[3].operator double();
+  param.finfoEps64 = args[4].operator double();
+  param.new_default = args[2].type_code() == kNull ? true : false;
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::LstsqParam>(&attrs);
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1]),
+                 NDArrayHandle(ndoutputs[2]),
+                 NDArrayHandle(ndoutputs[3])});
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_qr.cc b/src/api/operator/numpy/linalg/np_qr.cc
new file mode 100644
index 000000000000..e9c0ec5d66d3
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_qr.cc
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_qr.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_qr.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.qr")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_qr");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1])});
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_lstsq-inl.h b/src/operator/numpy/linalg/np_lstsq-inl.h
index 0389b7a5d92c..00fc19d9ed80 100644
--- a/src/operator/numpy/linalg/np_lstsq-inl.h
+++ b/src/operator/numpy/linalg/np_lstsq-inl.h
@@ -41,15 +41,34 @@ using namespace mshadow;
 
 struct LstsqParam : public dmlc::Parameter<LstsqParam> {
   double rcond;
+  float finfoEps32;
+  double finfoEps64;
   bool new_default;
   DMLC_DECLARE_PARAMETER(LstsqParam) {
     DMLC_DECLARE_FIELD(rcond)
     .set_default(-1)
     .describe("Cut-off ratio for small singular values");
+    DMLC_DECLARE_FIELD(finfoEps32)
+    .set_default(0)
+    .describe("Machine limits for float32 type");
+    DMLC_DECLARE_FIELD(finfoEps64)
+    .set_default(0)
+    .describe("Machine limits for float64 type");
     DMLC_DECLARE_FIELD(new_default)
     .set_default(false)
     .describe("Specifies whether rcond is default which is machine precision");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream rcond_s, finfoEps32_s, finfoEps64_s, new_default_s;
+    rcond_s << rcond;
+    finfoEps32_s << finfoEps32;
+    finfoEps64_s << finfoEps64;
+    new_default_s << new_default;
+    (*dict)["rcond"] = rcond_s.str();
+    (*dict)["finfoEps32"] = finfoEps32_s.str();
+    (*dict)["finfoEps64"] = finfoEps64_s.str();
+    (*dict)["new_default"] = new_default_s.str();
+  }
 };
 
 template<typename xpu, typename DType>
@@ -335,10 +354,12 @@ void LstsqOpForwardImpl(const TBlob& a,
                         const OpContext& ctx,
                         const std::vector<OpReqType>& req) {
   // Get param.
-  double rcond = nnvm::get<LstsqParam>(attrs.parsed).rcond;
-  bool new_default = nnvm::get<LstsqParam>(attrs.parsed).new_default;
+  const LstsqParam& param = nnvm::get<LstsqParam>(attrs.parsed);
+  double rcond = param.rcond;
+  bool new_default = param.new_default;
+  double finfoEps = a.type_flag_ == mshadow::kFloat32 ? param.finfoEps32 : param.finfoEps64;
   if (new_default) {
-    rcond *= std::max(a.shape_[0], a.shape_[1]);
+    rcond = finfoEps * std::max(a.shape_[0], a.shape_[1]);
   }
   const mxnet::TShape& a_shape = a.shape_;
   const mxnet::TShape& b_shape = b.shape_;

From da2adcfe050fabefa08f71cd4cb5fb68e95345de Mon Sep 17 00:00:00 2001
From: "D. Roberts" <denisa.roberts@denisaroberts.me>
Date: Thu, 16 Apr 2020 19:02:58 -0400
Subject: [PATCH 29/44] Add np.linalg.qr backward (#18050)

---
 src/operator/numpy/linalg/np_qr-inl.h  | 132 +++++++++++++++++++++++++
 src/operator/numpy/linalg/np_qr.cc     |  10 +-
 src/operator/numpy/linalg/np_qr.cu     |   3 +
 tests/python/unittest/test_numpy_op.py |  69 ++++++++++++-
 4 files changed, 208 insertions(+), 6 deletions(-)

diff --git a/src/operator/numpy/linalg/np_qr-inl.h b/src/operator/numpy/linalg/np_qr-inl.h
index c33e12059598..0f332e4a661a 100644
--- a/src/operator/numpy/linalg/np_qr-inl.h
+++ b/src/operator/numpy/linalg/np_qr-inl.h
@@ -475,6 +475,138 @@ void NumpyLaQrForward(const nnvm::NodeAttrs& attrs,
   QrOpForwardImpl<xpu>(a, q, r, req, workspace, ctx, attrs);
 }
 
+template<int req>
+struct assign_helper {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *in_data, DType *out_data) {
+    KERNEL_ASSIGN(out_data[i], req, in_data[i]);
+  }
+};
+
+struct qr_backward {
+  template<typename xpu, typename DType>
+  static void op(const Tensor<xpu, 3, DType>& dA,
+                 const Tensor<xpu, 3, DType>& dQ,
+                 const Tensor<xpu, 3, DType>& dR,
+                 const Tensor<xpu, 3, DType>& A,
+                 const Tensor<xpu, 3, DType>& Q,
+                 const Tensor<xpu, 3, DType>& R,
+                 const Tensor<xpu, 3, DType>& M,
+                 const OpContext& ctx, const nnvm::NodeAttrs& attrs) {
+    // Implements case m >= n; da = [dq + q@copyltu(M))]@r**(-T)
+    // Where M = r@(dr**T) - (dq**T)@q
+    // Reference: https://arxiv.org/abs/1710.08717
+    Stream<xpu> *s = ctx.get_stream<xpu>();
+    if (dQ.dptr_ != dA.dptr_) Copy(dA, dQ, s);
+    // M = R@dR_T
+    trmm::op(R, M, DType(1.0), false, false, false, s);
+    // M = R@dR_T - dQ_T@Q
+    gemm::op(dA, Q, M, DType(-1.0), DType(1.0), true, false, s);
+    // M = copyltu(M)
+    mxnet_op::Kernel<CopyTriangularToOppositeSide, xpu>::Launch
+      (s, M.MSize(), M.size(1) * M.stride_, M.stride_, M.dptr_, false);
+    // dA = dQ + Q@M
+    gemm::op(Q, M, dA, DType(1.0), DType(1.0), false, false, s);
+    // dA = dA@R_inv_T
+    trsm::op(R, dA, DType(1.0), true, false, true, s);
+  }
+};
+
+template<typename xpu>
+size_t QrBackwardWorkspaceSize(const TBlob& a,
+                               const TBlob& r,
+                               const TBlob& grad_a) {
+  if (0U == a.Size()) { return 0U; }
+
+  MSHADOW_SGL_DBL_TYPE_SWITCH(grad_a.type_flag_, DType, {
+    size_t work_space_size = 0;
+    // for grad a and M
+    work_space_size += a.Size();
+    work_space_size += r.Size();
+    return work_space_size * sizeof(DType);
+  });
+  LOG(FATAL) << "InternalError: cannot reach here";
+  return 0U;
+}
+
+template<typename xpu>
+void QrBackwardImpl(const TBlob& grad_a,
+                    const TBlob& grad_q,
+                    const TBlob& grad_r,
+                    const TBlob& a,
+                    const TBlob& q,
+                    const TBlob& r,
+                    const std::vector<OpReqType>& req,
+                    const Tensor<xpu, 1, char>& workspace,
+                    const OpContext& ctx,
+                    const nnvm::NodeAttrs& attrs) {
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const mxnet::TShape& a_shape = a.shape_;
+  const mxnet::TShape& r_shape = r.shape_;
+  const int a_ndim = a_shape.ndim();
+  const int n = a.size(a_ndim - 1);
+
+  if (kNullOp == req[0]) { return; }
+
+  if (0U == a_shape.Size()) { return; }
+
+  MSHADOW_SGL_DBL_TYPE_SWITCH(grad_a.type_flag_, DType, {
+    // case m >= n; Q of same shape with A and R is (n, n)
+    DType *m_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+    DType *grad_a_ptr = m_ptr + r_shape.Size();
+    TBlob temp_m(m_ptr, r_shape, xpu::kDevMask);
+    TBlob grad_a_data(grad_a_ptr, a_shape, xpu::kDevMask);
+    // dR_T
+    mxnet_op::Kernel<QrTypeTransposeHelper, xpu>::Launch(
+      s, r_shape.Size(), grad_r.dptr<DType>(), m_ptr, n, n, n * n);
+
+    qr_backward::op(grad_a_data.FlatToKD<xpu, 3, DType>(s),
+                    grad_q.FlatToKD<xpu, 3, DType>(s),
+                    grad_r.FlatToKD<xpu, 3, DType>(s),
+                    a.FlatToKD<xpu, 3, DType>(s),
+                    q.FlatToKD<xpu, 3, DType>(s),
+                    r.FlatToKD<xpu, 3, DType>(s),
+                    temp_m.FlatToKD<xpu, 3, DType>(s),
+                    ctx, attrs);
+
+    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+    mxnet_op::Kernel<assign_helper<req_type>, xpu>::Launch(
+      s, a_shape.Size(), grad_a_data.dptr<DType>(), grad_a.dptr<DType>());
+    });
+  });
+}
+
+// (dQ, dR, A, Q, R) => (dA)
+template<typename xpu>
+void NumpyLaQrBackward(const nnvm::NodeAttrs& attrs,
+                       const OpContext& ctx,
+                       const std::vector<TBlob>& inputs,
+                       const std::vector<OpReqType>& req,
+                       const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+  CHECK_EQ(inputs.size(), 5U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req.size(), 1U);
+
+  const TBlob& grad_q = inputs[0];
+  const TBlob& grad_r = inputs[1];
+  const TBlob& a = inputs[2];
+  const TBlob& q = inputs[3];
+  const TBlob& r = inputs[4];
+  const TBlob& grad_a = outputs[0];
+  const int a_ndim = a.shape_.ndim();
+  const int n = a.size(a_ndim - 1);
+  const int m = a.size(a_ndim - 2);
+
+  CHECK_LE(n, m)
+    << "QrBackward not implemented when ncols > nrows";
+
+  size_t workspace_size = QrBackwardWorkspaceSize<xpu>(a, r, grad_a);
+  Tensor<xpu, 1, char> workspace = ctx.requested[0]
+    .get_space_typed<xpu, 1, char>(Shape1(workspace_size), ctx.get_stream<xpu>());
+  QrBackwardImpl<xpu>(grad_a, grad_q, grad_r, a, q, r, req, workspace, ctx, attrs);
+}
+
 }  // namespace op
 }  // namespace mxnet
 
diff --git a/src/operator/numpy/linalg/np_qr.cc b/src/operator/numpy/linalg/np_qr.cc
index c4ef7f3ec783..6e72a1ab98ce 100644
--- a/src/operator/numpy/linalg/np_qr.cc
+++ b/src/operator/numpy/linalg/np_qr.cc
@@ -98,8 +98,16 @@ NNVM_REGISTER_OP(_npi_qr)
 .set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs) {
   return std::vector<ResourceRequest>{ResourceRequest::kTempSpace}; })
 .set_attr<FCompute>("FCompute<cpu>", NumpyLaQrForward<cpu>)
-.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseInOut{"_backward_npi_qr"})
 .add_argument("A", "NDArray-or-Symbol", "Input matrices to be factorized");
 
+NNVM_REGISTER_OP(_backward_npi_qr)
+.set_num_inputs(5)
+.set_num_outputs(1)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs) {
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace}; })
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FCompute>("FCompute<cpu>", NumpyLaQrBackward<cpu>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_qr.cu b/src/operator/numpy/linalg/np_qr.cu
index 5547d2627381..83d95584e104 100644
--- a/src/operator/numpy/linalg/np_qr.cu
+++ b/src/operator/numpy/linalg/np_qr.cu
@@ -34,6 +34,9 @@ namespace op {
 NNVM_REGISTER_OP(_npi_qr)
 .set_attr<FCompute>("FCompute<gpu>", NumpyLaQrForward<gpu>);
 
+NNVM_REGISTER_OP(_backward_npi_qr)
+.set_attr<FCompute>("FCompute<gpu>", NumpyLaQrBackward<gpu>);
+
 #endif
 
 }  // namespace op
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 4b21853c3433..06cbb355d130 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -4976,11 +4976,55 @@ def __init__(self):
         def hybrid_forward(self, F, data):
             return F.np.linalg.qr(data)
 
+    def get_expected_grad(a, q, r):
+        if 0 in r.shape:
+            return r
+        def copyltu(M):
+                # shape of M is [batch, m, m]
+                eye = _np.array([_np.eye(M.shape[-1]) for i in range(M.shape[0])])
+                lower = _np.tril(M) - eye * M
+                lower_mask = _np.tril(_np.ones_like(M))
+                ret = lower_mask * M + lower.swapaxes(-1, -2)
+                return ret
+        shape_r = r.shape
+        shape_q = q.shape
+        shape_a = a.shape
+        r = r.reshape(-1, shape_r[-2], shape_r[-1])
+        q = q.reshape(-1, shape_q[-2], shape_q[-1])
+        dq = _np.ones_like(q)
+        dr = _np.ones_like(r)
+        dq_t = dq.swapaxes(-1, -2)
+        dr_t = dr.swapaxes(-1, -2)
+        r_inv = _np.linalg.inv(r)
+        r_inv_t = r_inv.swapaxes(-1, -2)
+        r_t = r.swapaxes(-1, -2)
+        # Get M
+        M = _np.matmul(r, dr_t) - _np.matmul(dq_t, q)
+        da = _np.matmul(dq + _np.matmul(q, copyltu(M)), r_inv_t)
+        return da.reshape(a.shape)
+
+    def well_conditioned_rectang_matrix_2D(shape, max_cond=4):
+        m, n = shape[-2], shape[-1]
+        while 1:
+            M1 = _np.random.uniform(-10, 10, (m, n))
+            Q1, R1 = _np.linalg.qr(M1)
+            s = _np.ones(n)
+            D = _np.diag(s)
+            M2 =_np.random.uniform(-10, 10, (n, n))
+            Q2, R2 = _np.linalg.qr(M2)
+            a = _np.matmul(_np.matmul(Q1, D), _np.swapaxes(Q2, -1, -2))
+            if (_np.linalg.cond(a, 2) < max_cond):
+                return a
+
+    def well_conditioned_rectang_matrix_nD(shape, max_cond=4):
+        p = int(_np.prod(shape[:-2])) if len(shape) > 2 else 1
+        return _np.array([well_conditioned_rectang_matrix_2D(shape, max_cond) for i in range(p)]).reshape(shape)
+
     def check_qr(q, r, a_np):
         # check Q@R = A
         t = _np.matmul(q, r)
-        assert t.shape == data_np.shape
-        assert_almost_equal(t, data_np, rtol=rtol, atol=atol)
+        assert t.shape == a_np.shape
+        assert_almost_equal(t, a_np, rtol=rtol, atol=atol)
         # check QT@Q = I
         qT = _np.swapaxes(q, -2, -1)
         I = _np.matmul(qT, q)
@@ -5022,19 +5066,34 @@ def check_qr(q, r, a_np):
         (2, 3, 4, 3)
     ]
     dtypes = ['float32', 'float64']
-    for hybridize, shape, dtype in itertools.product([True, False], shapes, dtypes):
+    for hybridize, shape, dtype in itertools.product([False, True], shapes, dtypes):
         rtol = atol = 0.01
         test_qr = TestQR()
         if hybridize:
             test_qr.hybridize()
-        data_np = _np.random.uniform(-10.0, 10.0, shape)
+
+        if 0 in shape:
+            data_np = _np.ones(shape)
+        elif shape[-2] >= shape[-1]:
+            data_np = well_conditioned_rectang_matrix_nD(shape, max_cond=4)
+        else:
+            data_np = _np.random.uniform(-10.0, 10.0, shape)
         data_np = _np.array(data_np, dtype=dtype)
         data = np.array(data_np, dtype=dtype)
 
-        ret = test_qr(data)
+        data.attach_grad()
+        with mx.autograd.record():
+            ret = test_qr(data)
         Q, R = ret[0], ret[1]
         check_qr(Q, R, data_np)
 
+        # Only shapes m >= n have gradient
+        if 0 not in R.shape and shape[-2] >= shape[-1]:
+            assert data.grad.shape == data_np.shape
+            backward_expected = get_expected_grad(data_np, Q.asnumpy(), R.asnumpy())
+            mx.autograd.backward(ret)
+            assert_almost_equal(data.grad.asnumpy(), backward_expected, rtol=rtol, atol=atol)
+
         # check imperative once more; mode='reduced' is default
         # behavior and optional parameter in original numpy
         ret = np.linalg.qr(data, mode='reduced')

From 68a2a1aac4127c8c367f6ffed55c770381d99bc7 Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Wed, 8 Apr 2020 08:02:00 +0800
Subject: [PATCH 30/44] ffi_array_split, v/h/dsplit (#17873)

---
 benchmark/python/ffi/benchmark_ffi.py  |   4 +
 python/mxnet/ndarray/numpy/_op.py      |  44 ++-----
 python/mxnet/symbol/numpy/_symbol.py   |  23 ++--
 src/api/operator/numpy/np_matrix_op.cc | 154 +++++++++++++++++++++++++
 src/operator/tensor/matrix_op.cc       |   1 +
 5 files changed, 186 insertions(+), 40 deletions(-)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index d484f77ac5cb..c5fc2094f0a9 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -119,6 +119,10 @@ def prepare_workloads():
                            out=dnp.array([False, False], dtype=bool), keepdims=False)
     OpArgMngr.add_workload("roll", pool["2x2"], 1, axis=0)
     OpArgMngr.add_workload("rot90", pool["2x2"], 2)
+    OpArgMngr.add_workload("array_split", pool['2X2'], 2, axis=1)
+    OpArgMngr.add_workload("vsplit", pool['2X2'], 2)
+    OpArgMngr.add_workload("hsplit", pool['2X2'], 2)
+    OpArgMngr.add_workload("dsplit", pool['2X2x2'], 2)
 
 
 def benchmark_helper(f, *args, **kwargs):
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 61cc6b4a8f7e..50334b5763e3 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -3763,18 +3763,9 @@ def array_split(ary, indices_or_sections, axis=0):
     >>> np.array_split(x, 3)
     [array([0.,  1.,  2.]), array([3.,  4.]), array([5.,  6.])]
     """
-    indices = []
-    sections = 0
-    if isinstance(indices_or_sections, integer_types):
-        sections = indices_or_sections
-    elif isinstance(indices_or_sections, (list, set, tuple)):
-        indices = [0] + list(indices_or_sections)
-    else:
-        raise ValueError('indices_or_sections must be either int, or tuple / list / set of ints')
-    ret = _npi.split(ary, indices, axis, False, sections)
-    if not isinstance(ret, list):
-        return [ret]
-    return ret
+    if isinstance(indices_or_sections, set):
+        indices_or_sections = list(indices_or_sections)
+    return list(_api_internal.array_split(ary, indices_or_sections, axis))
 # pylint: enable=redefined-outer-name
 
 
@@ -3871,20 +3862,9 @@ def hsplit(ary, indices_or_sections):
     >>> np.hsplit(x, [2, 2])
     [array([0., 1.]), array([], dtype=float32), array([2., 3.])]
     """
-    if len(ary.shape) < 1:
-        raise ValueError('hsplit only works on arrays of 1 or more dimensions')
-    indices = []
-    sections = 0
-    if isinstance(indices_or_sections, integer_types):
-        sections = indices_or_sections
-    elif isinstance(indices_or_sections, (list, set, tuple)):
-        indices = [0] + list(indices_or_sections)
-    else:
-        raise ValueError('indices_or_sections must be either int, or tuple / list / set of ints')
-    ret = _npi.hsplit(ary, indices, 1, False, sections)
-    if not isinstance(ret, list):
-        return [ret]
-    return ret
+    if isinstance(indices_or_sections, set):
+        indices_or_sections = list(indices_or_sections)
+    return list(_api_internal.hsplit(ary, indices_or_sections))
 # pylint: enable=redefined-outer-name
 
 
@@ -3962,9 +3942,9 @@ def vsplit(ary, indices_or_sections):
             [6., 7.]]])]
 
     """
-    if len(ary.shape) < 2:
-        raise ValueError("vsplit only works on arrays of 2 or more dimensions")
-    return split(ary, indices_or_sections, 0)
+    if isinstance(indices_or_sections, set):
+        indices_or_sections = list(indices_or_sections)
+    return list(_api_internal.vsplit(ary, indices_or_sections))
 
 
 # pylint: disable=redefined-outer-name
@@ -4021,9 +4001,9 @@ def dsplit(ary, indices_or_sections):
             [15.]]]),
     array([], shape=(2, 2, 0), dtype=float64)]
     """
-    if len(ary.shape) < 3:
-        raise ValueError('dsplit only works on arrays of 3 or more dimensions')
-    return split(ary, indices_or_sections, 2)
+    if isinstance(indices_or_sections, set):
+        indices_or_sections = list(indices_or_sections)
+    return list(_api_internal.dsplit(ary, indices_or_sections))
 # pylint: enable=redefined-outer-name
 
 
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 576c2a5341a3..fff778a90f1d 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -3559,8 +3559,7 @@ def split(ary, indices_or_sections, axis=0):
         indices = [0] + list(indices_or_sections)
     else:
         raise ValueError('indices_or_sections must either int or tuple / list / set of ints')
-    ret = _npi.split(ary, indices, axis, False, sections)
-    return ret
+    return _npi.split(ary, indices, axis, False, sections)
 # pylint: enable=redefined-outer-name
 
 
@@ -3605,7 +3604,7 @@ def array_split(ary, indices_or_sections, axis=0):
         indices = [0] + list(indices_or_sections)
     else:
         raise ValueError('indices_or_sections must either int or tuple / list / set of ints')
-    ret = _npi.split(ary, indices, axis, False, sections)
+    ret = _npi.array_split(ary, indices, axis, False, sections)
     if not isinstance(ret, list):
         return [ret]
     return ret
@@ -3713,11 +3712,11 @@ def hsplit(ary, indices_or_sections):
         indices = [0] + list(indices_or_sections)
     else:
         raise ValueError('indices_or_sections must either int or tuple of ints')
-    ret = _npi.hsplit(ary, indices, 1, False, sections)
-    return ret
+    return _npi.hsplit(ary, indices, 1, False, sections)
 # pylint: enable=redefined-outer-name
 
 
+# pylint: disable=redefined-outer-name
 @set_module('mxnet.symbol.numpy')
 def vsplit(ary, indices_or_sections):
     r"""
@@ -3766,7 +3765,16 @@ def vsplit(ary, indices_or_sections):
     an error will be thrown.
 
     """
-    return split(ary, indices_or_sections, 0)
+    indices = []
+    sections = 0
+    if isinstance(indices_or_sections, int):
+        sections = indices_or_sections
+    elif isinstance(indices_or_sections, (list, set, tuple)):
+        indices = [0] + list(indices_or_sections)
+    else:
+        raise ValueError('indices_or_sections must either int or tuple of ints')
+    return _npi.split(ary, indices, 0, False, sections)
+# pylint: enable=redefined-outer-name
 
 
 # pylint: disable=redefined-outer-name
@@ -3804,8 +3812,7 @@ def dsplit(ary, indices_or_sections):
         indices = [0] + list(indices_or_sections)
     else:
         raise ValueError('indices_or_sections must either int or tuple of ints')
-    ret = _npi.dsplit(ary, indices, 2, False, sections)
-    return ret
+    return _npi.dsplit(ary, indices, 2, False, sections)
 # pylint: enable=redefined-outer-name
 
 
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index 09fb65f0261f..0c566b5d16ef 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -168,6 +168,160 @@ MXNET_REGISTER_API("_npi.rot90")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.array_split")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_array_split");
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = args[2].operator int();
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_inputs = 1;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+MXNET_REGISTER_API("_npi.dsplit")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_split");
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  CHECK_GE(inputs[0]->shape().ndim(), 3)
+      << "ValueError: dsplit only works on arrays of 3 or more dimensions";
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = 2;
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_EQ(inputs[0]->shape()[2] % param.sections, 0)
+      << "ValueError: array split does not result in an equal division";
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+MXNET_REGISTER_API("_npi.hsplit")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_hsplit");
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  CHECK_GE(inputs[0]->shape().ndim(), 1)
+    << "ValueError: hsplit only works on arrays of 1 or more dimensions";
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = 0;
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
+MXNET_REGISTER_API("_npi.vsplit")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_split");
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  CHECK_GE(inputs[0]->shape().ndim(), 2)
+      << "ValueError: vsplit only works on arrays of 2 or more dimensions";
+  nnvm::NodeAttrs attrs;
+  op::SplitParam param;
+  param.axis = 0;
+  param.squeeze_axis = false;
+  if (args[1].type_code() == kDLInt) {
+    param.indices = TShape(0, 0);
+    param.sections = args[1].operator int();
+    CHECK_EQ(inputs[0]->shape()[0] % param.sections, 0)
+      << "ValueError: array split does not result in an equal division";
+    CHECK_GT(param.sections, 0)
+      << "ValueError: number sections must be larger than 0";
+  } else {
+    TShape t = TShape(args[1].operator ObjectRef());
+    param.indices = TShape(t.ndim() + 1, 0);
+    for (int i = 0; i < t.ndim(); ++i) {
+      param.indices[i + 1] = t[i];
+    }
+    param.sections = 0;
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::SplitParam>(&attrs);
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  std::vector<NDArrayHandle> ndarray_handles;
+  ndarray_handles.reserve(num_outputs);
+  for (int i = 0; i < num_outputs; ++i) {
+    ndarray_handles.emplace_back(ndoutputs[i]);
+  }
+  *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
+});
+
 MXNET_REGISTER_API("_npi.diag")
 .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
   using namespace runtime;
diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc
index 9e63730ec001..ad6b44b02f44 100644
--- a/src/operator/tensor/matrix_op.cc
+++ b/src/operator/tensor/matrix_op.cc
@@ -1038,6 +1038,7 @@ Example::
 
 NNVM_REGISTER_OP(_split_v2)
 .add_alias("_npi_split")
+.add_alias("_npi_array_split")
 .describe(R"code(Splits an array along a particular axis into multiple sub-arrays.
 Example::
    x  = [[[ 1.]

From f998d0ebc93adb7fb4c5bf421223ffb18bacdb4a Mon Sep 17 00:00:00 2001
From: JiangZhaoh <54654391+JiangZhaoh@users.noreply.github.com>
Date: Fri, 10 Apr 2020 14:01:15 +0800
Subject: [PATCH 31/44] [numpy] FFI for insert \ delete \ matmul etc. (#17759)

* insert - ffi

* delete - ffi

* resolve comment & fix sanity error

fix compile warning

fix compile error

* matmul ffi & rebase

* fix sanity error

* concatenate ffi & append ffi

* fix conflict

* arange ffi

* blackman ffi

* benchmark

* hamming ffi & hanning ffi & benchmark

* logspace ffi &linspace ffi

* mean ffi

* gamma ffi

* radom.normal ffi

* random.uniform ffi

* fix sanity error

* resolve comments

* fix rebase sanity error

fix sanity error

fix sanity error

* fix CI error

* fix compile error

* try to pass CI

* resolve comment

* rebase
---
 benchmark/python/ffi/benchmark_ffi.py         |  15 ++
 python/mxnet/ndarray/numpy/_op.py             |  87 ++++++----
 python/mxnet/ndarray/numpy/random.py          |  67 +++-----
 .../numpy/np_broadcast_reduce_op_value.cc     |  40 ++++-
 src/api/operator/numpy/np_delete_op.cc        | 101 ++++++++++++
 src/api/operator/numpy/np_init_op.cc          | 126 +++++++++++++-
 src/api/operator/numpy/np_insert_op.cc        | 156 ++++++++++++++++++
 src/api/operator/numpy/np_matmul_op.cc        |  51 ++++++
 src/api/operator/numpy/np_matrix_op.cc        |  32 ++++
 src/api/operator/numpy/np_window_op.cc        |  79 +++++++++
 src/api/operator/random/np_gamma_op.cc        | 108 ++++++++++++
 src/api/operator/random/np_normal_op.cc       |  96 +++++++++++
 src/api/operator/random/np_uniform_op.cc      |  96 +++++++++++
 src/operator/numpy/np_broadcast_reduce_op.h   |  16 ++
 src/operator/numpy/np_delete_op-inl.h         |  14 ++
 src/operator/numpy/np_init_op.h               |  26 +++
 src/operator/numpy/np_insert_op-inl.h         |  16 ++
 src/operator/numpy/np_matrix_op-inl.h         |   7 +
 src/operator/numpy/np_window_op.h             |   8 +
 src/operator/numpy/random/np_gamma_op.h       |  12 ++
 src/operator/numpy/random/np_normal_op.h      |  12 ++
 src/operator/numpy/random/np_uniform_op.h     |  12 ++
 src/operator/tensor/init_op.h                 |  29 ++++
 23 files changed, 1128 insertions(+), 78 deletions(-)
 create mode 100644 src/api/operator/numpy/np_delete_op.cc
 create mode 100644 src/api/operator/numpy/np_insert_op.cc
 create mode 100644 src/api/operator/numpy/np_matmul_op.cc
 create mode 100644 src/api/operator/numpy/np_window_op.cc
 create mode 100644 src/api/operator/random/np_gamma_op.cc
 create mode 100644 src/api/operator/random/np_normal_op.cc
 create mode 100644 src/api/operator/random/np_uniform_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index c5fc2094f0a9..2ab3226d967e 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -123,6 +123,21 @@ def prepare_workloads():
     OpArgMngr.add_workload("vsplit", pool['2X2'], 2)
     OpArgMngr.add_workload("hsplit", pool['2X2'], 2)
     OpArgMngr.add_workload("dsplit", pool['2X2x2'], 2)
+    OpArgMngr.add_workload("arange", 10)
+    OpArgMngr.add_workload("concatenate", (pool['1x2'], pool['1x2'], pool['1x2']), axis=0)
+    OpArgMngr.add_workload("append", pool['2x2'], pool['1x2'], axis=0)
+    OpArgMngr.add_workload("insert", pool['3x2'], 1, pool['1x1'], axis=0)
+    OpArgMngr.add_workload("delete", pool['3x2'], 1, axis=0)
+    OpArgMngr.add_workload("blackman", 12)
+    OpArgMngr.add_workload("eye", 5)
+    OpArgMngr.add_workload("hamming", 12)
+    OpArgMngr.add_workload("hanning", 12)
+    OpArgMngr.add_workload("linspace", 0, 10, 8, endpoint=False)
+    OpArgMngr.add_workload("logspace", 2.0, 3.0, num=4, base=2.0, dtype=onp.float32)
+    OpArgMngr.add_workload("matmul", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("mean", pool['2x2'], axis=0, keepdims=True)
+    OpArgMngr.add_workload("random.gamma", 1, size=(2, 3))
+    OpArgMngr.add_workload("random.normal", 1, size=(2, 3))
 
 
 def benchmark_helper(f, *args, **kwargs):
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 50334b5763e3..7192f0e5824c 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -549,8 +549,12 @@ def arange(start, stop=None, step=1, dtype=None, ctx=None):
     """
     if dtype is None:
         dtype = 'float32'
+    if dtype is not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if stop is None:
         stop = start
         start = 0
@@ -560,7 +564,7 @@ def arange(start, stop=None, step=1, dtype=None, ctx=None):
         raise ValueError('start and stop cannot be both None')
     if step == 0:
         raise ZeroDivisionError('step cannot be 0')
-    return _npi.arange(start=start, stop=stop, step=step, dtype=dtype, ctx=ctx)
+    return _api_internal.arange(start, stop, step, dtype, ctx)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -776,11 +780,11 @@ def insert(arr, obj, values, axis=None):
             start = obj.start
             stop = obj.stop
             step = 1 if obj.step is None else obj.step
-            return _npi.insert_slice(arr, val=values, start=start, stop=stop, step=step, axis=axis)
+            return _api_internal.insert_slice(arr, values, start, stop, step, axis)
         elif isinstance(obj, integer_types):
-            return _npi.insert_scalar(arr, val=values, int_ind=obj, axis=axis)
+            return _api_internal.insert_scalar(arr, values, obj, axis)
         elif isinstance(obj, NDArray):
-            return _npi.insert_tensor(arr, obj, val=values, axis=axis)
+            return _api_internal.insert_tensor(arr, obj, values, axis)
 
     if not isinstance(arr, NDArray):
         raise TypeError("'arr' can not support type {}".format(str(type(arr))))
@@ -790,11 +794,11 @@ def insert(arr, obj, values, axis=None):
         start = obj.start
         stop = obj.stop
         step = 1 if obj.step is None else obj.step
-        return _npi.insert_slice(arr, values, start=start, stop=stop, step=step, axis=axis)
+        return _api_internal.insert_slice(arr, values, start, stop, step, axis)
     elif isinstance(obj, integer_types):
-        return _npi.insert_scalar(arr, values, int_ind=obj, axis=axis)
+        return _api_internal.insert_scalar(arr, values, obj, axis)
     elif isinstance(obj, NDArray):
-        return _npi.insert_tensor(arr, values, obj, axis=axis)
+        return _api_internal.insert_tensor(arr, values, obj, axis)
     else:
         raise TypeError("'obj' can not support type {}".format(str(type(obj))))
 
@@ -1219,11 +1223,11 @@ def delete(arr, obj, axis=None):
         start = obj.start
         stop = obj.stop
         step = 1 if obj.step is None else obj.step
-        return _npi.delete(arr, start=start, stop=stop, step=step, axis=axis)
+        return _api_internal.delete(arr, start, stop, step, axis)
     elif isinstance(obj, integer_types):
-        return _npi.delete(arr, int_ind=obj, axis=axis)
+        return _api_internal.delete(arr, obj, axis)
     elif isinstance(obj, NDArray):
-        return _npi.delete(arr, obj, axis=axis)
+        return _api_internal.delete(arr, obj, axis)
     else:
         raise TypeError("'obj' can not support type {}".format(str(type(obj))))
 
@@ -1325,7 +1329,7 @@ def matmul(a, b, out=None):
     ...
     mxnet.base.MXNetError: ... : Multiplication by scalars is not allowed.
     """
-    return _npi.matmul(a, b, out=out)
+    return _api_internal.matmul(a, b, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1742,8 +1746,12 @@ def eye(N, M=None, k=0, dtype=_np.float32, **kwargs):
     _sanity_check_params('eye', ['order'], kwargs)
     ctx = kwargs.pop('ctx', current_context())
     if ctx is None:
-        ctx = current_context()
-    return _npi.eye(N, M, k, ctx, dtype)
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.eye(N, M, k, ctx, dtype)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1835,12 +1843,16 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis
     if axis != 0:
         raise NotImplementedError("the function only support axis 0")
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
     if retstep:
         step = (stop - start) / (num - 1)
-        return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype), step
+        return _api_internal.linspace(start, stop, num, endpoint, ctx, dtype), step
     else:
-        return _npi.linspace(start=start, stop=stop, num=num, endpoint=endpoint, ctx=ctx, dtype=dtype)
+        return _api_internal.linspace(start, stop, num, endpoint, ctx, dtype)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1922,8 +1934,12 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0,
     if axis != 0:
         raise NotImplementedError("the function only support axis 0")
     if ctx is None:
-        ctx = current_context()
-    return _npi.logspace(start=start, stop=stop, num=num, endpoint=endpoint, base=base, ctx=ctx, dtype=dtype)
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.logspace(start, stop, num, endpoint, base, ctx, dtype)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4046,7 +4062,7 @@ def concatenate(seq, axis=0, out=None):
     array([[1., 2., 5.],
            [3., 4., 6.]])
     """
-    return _npi.concatenate(*seq, axis=axis, out=out)
+    return _api_internal.concatenate(*seq, axis, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4086,7 +4102,8 @@ def append(arr, values, axis=None):  # pylint: disable=redefined-outer-name
            [4., 5., 6.],
            [7., 8., 9.]])
     """
-    return _npi.concatenate(arr, values, axis=axis, out=None)
+    out = None
+    return _api_internal.concatenate(arr, values, axis, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4722,7 +4739,9 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable
     >>> np.mean(a, dtype=np.float64)
     array(0.55)
     """
-    return _npi.mean(a, axis=axis, dtype=dtype, keepdims=keepdims, out=out)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.mean(a, axis, dtype, keepdims, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5228,8 +5247,12 @@ def hanning(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     if ctx is None:
-        ctx = current_context()
-    return _npi.hanning(M, dtype=dtype, ctx=ctx)
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.hanning(M, dtype, ctx)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5308,8 +5331,12 @@ def hamming(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     if ctx is None:
-        ctx = current_context()
-    return _npi.hamming(M, dtype=dtype, ctx=ctx)
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.hamming(M, dtype, ctx)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5386,8 +5413,12 @@ def blackman(M, dtype=_np.float32, ctx=None):
     >>> plt.show()
     """
     if ctx is None:
-        ctx = current_context()
-    return _npi.blackman(M, dtype=dtype, ctx=ctx)
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = _np.dtype(dtype).name
+    return _api_internal.blackman(M, dtype, ctx)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/python/mxnet/ndarray/numpy/random.py b/python/mxnet/ndarray/numpy/random.py
index 3952a6fdb561..a25096a65224 100644
--- a/python/mxnet/ndarray/numpy/random.py
+++ b/python/mxnet/ndarray/numpy/random.py
@@ -19,8 +19,8 @@
 import numpy as np
 from ...context import current_context
 from . import _internal as _npi
-from ..ndarray import NDArray
 from . import _api_internal
+from ..ndarray import NDArray
 
 
 __all__ = ['randint', 'uniform', 'normal', "choice", "rand", "multinomial", "multivariate_normal",
@@ -123,26 +123,17 @@ def uniform(low=0.0, high=1.0, size=None, dtype=None, ctx=None, out=None):
     out : ndarray
         Drawn samples from the parameterized uniform distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(low, np_ndarray), isinstance(high, np_ndarray))
     if dtype is None:
         dtype = 'float32'
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = np.dtype(dtype).name
     if size == ():
         size = None
-    if input_type == (True, True):
-        return _npi.uniform(low, high, low=None, high=None, size=size,
-                            ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (False, True):
-        return _npi.uniform(high, low=low, high=None, size=size,
-                            ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (True, False):
-        return _npi.uniform(low, low=None, high=high, size=size,
-                            ctx=ctx, dtype=dtype, out=out)
-    else:
-        return _npi.uniform(low=low, high=high, size=size,
-                            ctx=ctx, dtype=dtype, out=out)
+    return _api_internal.uniform(low, high, size, ctx, dtype, out)
 
 
 def normal(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):
@@ -174,26 +165,17 @@ def normal(loc=0.0, scale=1.0, size=None, dtype=None, ctx=None, out=None):
     out : ndarray
         Drawn samples from the parameterized normal distribution.
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(loc, np_ndarray), isinstance(scale, np_ndarray))
     if dtype is None:
         dtype = 'float32'
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = np.dtype(dtype).name
     if size == ():
         size = None
-    if input_type == (True, True):
-        return _npi.normal(loc, scale, loc=None, scale=None, size=size,
-                           ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (False, True):
-        return _npi.normal(scale, loc=loc, scale=None, size=size,
-                           ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (True, False):
-        return _npi.normal(loc, loc=None, scale=scale, size=size,
-                           ctx=ctx, dtype=dtype, out=out)
-    else:
-        return _npi.normal(loc=loc, scale=scale, size=size,
-                           ctx=ctx, dtype=dtype, out=out)
+    return _api_internal.normal(loc, scale, size, ctx, dtype, out)
 
 
 def lognormal(mean=0.0, sigma=1.0, size=None, dtype=None, ctx=None, out=None):
@@ -745,30 +727,19 @@ def gamma(shape, scale=1.0, size=None, dtype=None, ctx=None, out=None):
     electronic components, and arises naturally in processes for which the
     waiting times between Poisson distributed events are relevant.
     """
-    from ...numpy import ndarray as np_ndarray
-    input_type = (isinstance(shape, np_ndarray), isinstance(scale, np_ndarray))
     if dtype is None:
         dtype = 'float32'
-    if ctx is None:
-        ctx = current_context()
     if out is not None:
         size = out.shape
     if size == ():
         size = None
-    if input_type == (True, True):
-        return _npi.gamma(shape, scale, shape=None, scale=None, size=size,
-                          ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (False, True):
-        return _npi.gamma(scale, shape=shape, scale=None, size=size,
-                          ctx=ctx, dtype=dtype, out=out)
-    elif input_type == (True, False):
-        return _npi.gamma(shape, shape=None, scale=scale, size=size,
-                          ctx=ctx, dtype=dtype, out=out)
+    if ctx is None:
+        ctx = str(current_context())
     else:
-        return _npi.gamma(shape=shape, scale=scale, size=size,
-                          ctx=ctx, dtype=dtype, out=out)
-
-    raise ValueError("Distribution parameters must be either mxnet.numpy.ndarray or numbers")
+        ctx = str(ctx)
+    if dtype is not None and not isinstance(dtype, str):
+        dtype = np.dtype(dtype).name
+    return _api_internal.gamma(shape, scale, size, ctx, dtype, out)
 
 
 def beta(a, b, size=None, dtype=None, ctx=None):
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
index 224451c70570..c2d87a285cde 100644
--- a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
@@ -25,7 +25,7 @@
 #include <mxnet/api_registry.h>
 #include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
-#include "../../../operator/tensor/broadcast_reduce_op.h"
+#include "../../../operator/numpy/np_broadcast_reduce_op.h"
 
 namespace mxnet {
 
@@ -51,4 +51,42 @@ MXNET_REGISTER_API("_npi.broadcast_to")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.mean")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_mean");
+  nnvm::NodeAttrs attrs;
+  op::NumpyReduceAxesParam param;
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::optional<mxnet::Tuple<int>>();
+  } else {
+    param.axis = mxnet::Tuple<int>(args[1].operator ObjectRef());
+  }
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::optional<int>();
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+  if (args[3].type_code() == kNull) {
+    param.keepdims = false;
+  } else {
+    param.keepdims = args[3].operator bool();
+  }
+  param.initial = dmlc::optional<double>();
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyReduceAxesParam>(&attrs);
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  NDArray* out = args[4].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(4);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
 }  // namespace mxnet
diff --git a/src/api/operator/numpy/np_delete_op.cc b/src/api/operator/numpy/np_delete_op.cc
new file mode 100644
index 000000000000..925c8b568e28
--- /dev/null
+++ b/src/api/operator/numpy/np_delete_op.cc
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_delete_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_delete_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/np_delete_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.delete")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_delete");
+  nnvm::NodeAttrs attrs;
+  op::NumpyDeleteParam param;
+  int num_inputs = 0;
+  param.start = dmlc::nullopt;
+  param.step = dmlc::nullopt;
+  param.stop = dmlc::nullopt;
+  param.int_ind = dmlc::nullopt;
+  param.axis = dmlc::nullopt;
+  if (args.num_args == 3) {
+    if (args[1].type_code() == kDLInt ||
+        args[1].type_code() == kDLFloat) {
+      if (args[1].type_code() == kDLInt) {
+        param.int_ind = args[1].operator int();
+      } else if (args[1].type_code() == kDLFloat) {
+        param.int_ind = static_cast<int>(args[1].operator double());
+      }
+      if (args[2].type_code() == kDLInt) {
+        param.axis = args[2].operator int();
+      } else if (args[2].type_code() == kDLFloat) {
+        param.axis = static_cast<int>(args[2].operator double());
+      }
+      num_inputs = 1;
+    } else {
+      if (args[2].type_code() == kDLInt) {
+        param.axis = args[2].operator int();
+      } else if (args[2].type_code() == kDLFloat) {
+        param.axis = static_cast<int>(args[2].operator double());
+      }
+      num_inputs = 2;
+    }
+  } else {
+    num_inputs = 1;
+    if (args[1].type_code() == kDLInt) {
+      param.start = args[1].operator int();
+    } else if (args[1].type_code() == kDLFloat) {
+      param.start = static_cast<int>(args[1].operator double());
+    }
+    if (args[2].type_code() == kDLInt) {
+      param.stop = args[2].operator int();
+    } else if (args[2].type_code() == kDLFloat) {
+      param.stop = static_cast<int>(args[2].operator double());
+    }
+    if (args[3].type_code() == kDLInt) {
+      param.step = args[3].operator int();
+    } else if (args[3].type_code() == kDLFloat) {
+      param.step = static_cast<int>(args[3].operator double());
+    }
+    if (args[4].type_code() == kDLInt) {
+      param.axis = args[4].operator int();
+    } else if (args[4].type_code() == kDLFloat) {
+      param.axis = static_cast<int>(args[4].operator double());
+    }
+  }
+  std::vector<NDArray*> inputs;
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyDeleteParam>(&attrs);
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_init_op.cc b/src/api/operator/numpy/np_init_op.cc
index 147f8a6b2022..b61914917d7f 100644
--- a/src/api/operator/numpy/np_init_op.cc
+++ b/src/api/operator/numpy/np_init_op.cc
@@ -86,7 +86,6 @@ MXNET_REGISTER_API("_npi.full_like")
   } else {
     *ret = ndoutputs[0];
   }
-  *ret = ndoutputs[0];
 });
 
 MXNET_REGISTER_API("_npi.indices")
@@ -201,4 +200,129 @@ MXNET_REGISTER_API("_npi.atleast_3d")
   *ret = ADT(0, ndarray_handles.begin(), ndarray_handles.end());
 });
 
+MXNET_REGISTER_API("_npi.arange")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_arange");
+  nnvm::NodeAttrs attrs;
+  op::RangeParam param;
+  param.start = args[0].operator double();
+  if (args[1].type_code() == kNull) {
+    param.stop = dmlc::nullopt;
+  } else {
+    param.stop = args[1].operator double();
+  }
+  param.step = args[2].operator double();
+  param.repeat = 1;
+  param.infer_range = false;
+  if (args[3].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[3].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::RangeParam>(&attrs);
+  if (args[4].type_code() != kNull) {
+    attrs.dict["ctx"] = args[4].operator std::string();
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.eye")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_eye");
+  nnvm::NodeAttrs attrs;
+  op::NumpyEyeParam param;
+  param.N = args[0].operator nnvm::dim_t();
+  if (args[1].type_code() == kNull) {
+    param.M = dmlc::nullopt;
+  } else {
+    param.M = args[1].operator nnvm::dim_t();
+  }
+  param.k = args[2].operator nnvm::dim_t();
+  if (args[4].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyEyeParam>(&attrs);
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.linspace")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_linspace");
+  nnvm::NodeAttrs attrs;
+  op::LinspaceParam param;
+  param.start = args[0].operator double();
+  param.stop = args[1].operator double();
+  param.num = args[2].operator int();
+  if (args[3].type_code() == kNull) {
+    param.endpoint = true;
+  } else {
+    param.endpoint = args[3].operator bool();
+  }
+  if (args[5].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[5].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::LinspaceParam>(&attrs);
+  if (args[4].type_code() != kNull) {
+    attrs.dict["ctx"] = args[4].operator std::string();
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.logspace")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_logspace");
+  nnvm::NodeAttrs attrs;
+  op::LogspaceParam param;
+  param.start = args[0].operator double();
+  param.stop = args[1].operator double();
+  param.num = args[2].operator int();
+  if (args[3].type_code() == kNull) {
+    param.endpoint = true;
+  } else {
+    param.endpoint = args[3].operator bool();
+  }
+  if (args[4].type_code() == kNull) {
+    param.base = 10.0;
+  } else {
+    param.base = args[4].operator double();
+  }
+  if (args[6].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[6].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::LogspaceParam>(&attrs);
+  if (args[5].type_code() != kNull) {
+    attrs.dict["ctx"] = args[5].operator std::string();
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
 }  // namespace mxnet
diff --git a/src/api/operator/numpy/np_insert_op.cc b/src/api/operator/numpy/np_insert_op.cc
new file mode 100644
index 000000000000..0de645e91913
--- /dev/null
+++ b/src/api/operator/numpy/np_insert_op.cc
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_insert_op.cc
+ * \brief Implementation of the API of functions in the following file:
+          src/operator/numpy/np_insert_op_scalar.cc
+          src/operator/numpy/np_insert_op_slice.cc
+          src/operator/numpy/np_insert_op_tensor.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/np_insert_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.insert_scalar")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_insert_scalar");
+  nnvm::NodeAttrs attrs;
+  op::NumpyInsertParam param;
+  int num_inputs = 0;
+  param.start = dmlc::nullopt;
+  param.step = dmlc::nullopt;
+  param.stop = dmlc::nullopt;
+  if (args[1].type_code() == kDLInt ||
+      args[1].type_code() == kDLUInt ||
+      args[1].type_code() == kDLFloat) {
+    param.val = args[1].operator double();
+    num_inputs = 1;
+  } else {
+    param.val = dmlc::nullopt;
+    num_inputs = 2;
+  }
+  param.int_ind = args[2].operator int();
+  if (args[3].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[3].operator int();
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyInsertParam>(&attrs);
+  std::vector<NDArray*> inputs;
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.insert_slice")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_insert_slice");
+  nnvm::NodeAttrs attrs;
+  op::NumpyInsertParam param;
+  int num_inputs = 0;
+  if (args[1].type_code() == kDLInt ||
+      args[1].type_code() == kDLUInt ||
+      args[1].type_code() == kDLFloat) {
+    param.val = args[1].operator double();
+    num_inputs = 1;
+  } else {
+    param.val = dmlc::nullopt;
+    num_inputs = 2;
+  }
+  if (args[2].type_code() == kNull) {
+    param.start = dmlc::nullopt;
+  } else {
+    param.start = args[2].operator int();
+  }
+  if (args[3].type_code() == kNull) {
+    param.stop = dmlc::nullopt;
+  } else {
+    param.stop = args[3].operator int();
+  }
+  if (args[4].type_code() == kNull) {
+    param.step = dmlc::nullopt;
+  } else {
+    param.step = args[4].operator int();
+  }
+  if (args[5].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[5].operator int();
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyInsertParam>(&attrs);
+  std::vector<NDArray*> inputs;
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+MXNET_REGISTER_API("_npi.insert_tensor")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_insert_tensor");
+  nnvm::NodeAttrs attrs;
+  op::NumpyInsertParam param;
+  param.start = dmlc::nullopt;
+  param.step = dmlc::nullopt;
+  param.stop = dmlc::nullopt;
+  int num_inputs = 0;
+  if (args[2].type_code() == kDLInt ||
+      args[2].type_code() == kDLUInt ||
+      args[2].type_code() == kDLFloat) {
+    param.val = args[2].operator double();
+    num_inputs = 2;
+  } else {
+    param.val = dmlc::nullopt;
+    num_inputs = 3;
+  }
+  if (args[3].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[3].operator int();
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyInsertParam>(&attrs);
+  std::vector<NDArray*> inputs;
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matmul_op.cc b/src/api/operator/numpy/np_matmul_op.cc
new file mode 100644
index 000000000000..48f4ec06fe83
--- /dev/null
+++ b/src/api/operator/numpy/np_matmul_op.cc
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_matmul_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_matmul_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_matmul_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.matmul")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_matmul");
+  nnvm::NodeAttrs attrs;
+  int num_inputs = 2;
+  NDArray* inputs[2] = {args[0].operator mxnet::NDArray*(),
+                        args[1].operator mxnet::NDArray*()};
+  attrs.op = op;
+  NDArray* out = args[2].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(2);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_matrix_op.cc b/src/api/operator/numpy/np_matrix_op.cc
index 0c566b5d16ef..c7f98a630888 100644
--- a/src/api/operator/numpy/np_matrix_op.cc
+++ b/src/api/operator/numpy/np_matrix_op.cc
@@ -50,6 +50,38 @@ MXNET_REGISTER_API("_npi.expand_dims")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.concatenate")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_concatenate");
+  nnvm::NodeAttrs attrs;
+  op::NumpyConcatenateParam param;
+  int arg_size = args.num_args;
+  param.num_args = arg_size - 2;
+  if (args[arg_size - 2].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    param.axis = args[arg_size - 2].operator int();
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyConcatenateParam>(&attrs);
+  int num_inputs = arg_size - 2;
+  std::vector<NDArray*> inputs;
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  NDArray* out = args[arg_size - 1].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(arg_size - 1);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
 MXNET_REGISTER_API("_npi.dstack")
 .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
   using namespace runtime;
diff --git a/src/api/operator/numpy/np_window_op.cc b/src/api/operator/numpy/np_window_op.cc
new file mode 100644
index 000000000000..8800b5cf2d01
--- /dev/null
+++ b/src/api/operator/numpy/np_window_op.cc
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_window_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_window_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_window_op.h"
+
+namespace mxnet {
+
+inline static void SetNumpyWindowsParam(runtime::MXNetArgs args,
+                                        runtime::MXNetRetValue* ret,
+                                        const nnvm::Op* op) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  op::NumpyWindowsParam param;
+  if (args[0].type_code() == kNull) {
+    param.M = dmlc::nullopt;
+  } else {
+    param.M = args[0].operator nnvm::dim_t();
+  }
+  if (args[1].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[1].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::NumpyWindowsParam>(&attrs);
+  if (args[2].type_code() != kNull) {
+    attrs.dict["ctx"] = args[2].operator std::string();
+  }
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 0, nullptr, &num_outputs, nullptr);
+  *ret = ndoutputs[0];
+}
+
+MXNET_REGISTER_API("_npi.blackman")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_blackman");
+  SetNumpyWindowsParam(args, ret, op);
+});
+
+MXNET_REGISTER_API("_npi.hamming")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_hamming");
+  SetNumpyWindowsParam(args, ret, op);
+});
+
+MXNET_REGISTER_API("_npi.hanning")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_hanning");
+  SetNumpyWindowsParam(args, ret, op);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/random/np_gamma_op.cc b/src/api/operator/random/np_gamma_op.cc
new file mode 100644
index 000000000000..ec574273e6a3
--- /dev/null
+++ b/src/api/operator/random/np_gamma_op.cc
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_gamma_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_gamma_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/random/np_gamma_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.gamma")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_gamma");
+  nnvm::NodeAttrs attrs;
+  op::NumpyGammaParam param;
+  int num_inputs = 0;
+  std::vector<NDArray*> inputs;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    if (args[0].type_code() == kNull) {
+      param.shape = dmlc::nullopt;
+    } else {
+      param.shape = args[0].operator double();
+    }
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // both 'shape' and 'scale' are numeric types
+      num_inputs = 0;
+      if (args[1].type_code() == kNull) {
+        param.scale = dmlc::nullopt;
+      } else {
+        param.scale = args[1].operator double();
+      }
+    } else {
+      // 'shape' is numeric types but 'scale' is not
+      num_inputs = 1;
+      param.scale = dmlc::nullopt;
+      inputs.push_back(args[1].operator mxnet::NDArray*());
+    }
+  } else {
+    param.shape = dmlc::nullopt;
+    inputs.push_back(args[0].operator mxnet::NDArray*());
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // 'shape' is not numeric types but 'scale' is numeric types
+      num_inputs = 1;
+      if (args[1].type_code() == kNull) {
+        param.scale = dmlc::nullopt;
+      } else {
+        param.scale = args[1].operator double();
+      }
+    } else {
+      // nither 'shape' or 'scale' is numeric types
+      num_inputs = 2;
+      param.scale = dmlc::nullopt;
+      inputs.push_back(args[1].operator mxnet::NDArray*());
+    }
+  }
+  if (args[2].type_code() == kNull) {
+    param.size = dmlc::optional<mxnet::Tuple<int>>();
+  } else if (args[2].type_code() == kDLInt ||
+             args[2].type_code() == kDLFloat) {
+    param.size = Tuple<int>(1, args[2].operator int64_t());
+  } else {
+    param.size = Tuple<int>(args[2].operator ObjectRef());
+  }
+  if (args[4].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());
+  }
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  SetAttrDict<op::NumpyGammaParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/random/np_normal_op.cc b/src/api/operator/random/np_normal_op.cc
new file mode 100644
index 000000000000..a45936d21333
--- /dev/null
+++ b/src/api/operator/random/np_normal_op.cc
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_normal_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_normal_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/random/np_normal_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.normal")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_normal");
+  nnvm::NodeAttrs attrs;
+  op::NumpyNormalParam param;
+  int num_inputs = 0;
+  std::vector<NDArray*> inputs;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // 'loc' and 'scale' are both numeric types
+      num_inputs = 0;
+      param.loc = args[0].operator double();
+      param.scale = args[1].operator double();
+    } else {
+      // 'loc' is numeric types but 'scale' is not numeric types
+      num_inputs = 1;
+      param.loc = args[0].operator double();
+      param.scale = dmlc::nullopt;
+    }
+  } else {
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // 'loc' is not numeric types but 'scale' is numeric types
+      num_inputs = 1;
+      param.loc = dmlc::nullopt;
+      param.scale = args[1].operator double();
+    } else {
+      // nither 'loc' or 'scale' is numeric types
+      num_inputs = 2;
+    }
+  }
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  if (args[2].type_code() == kNull) {
+    param.size = dmlc::optional<mxnet::Tuple<int>>();
+  } else if (args[2].type_code() == kDLInt ||
+             args[2].type_code() == kDLFloat) {
+    param.size = Tuple<int>(1, args[2].operator int64_t());
+  } else {
+    param.size = Tuple<int>(args[2].operator ObjectRef());
+  }
+  if (args[4].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  SetAttrDict<op::NumpyNormalParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/random/np_uniform_op.cc b/src/api/operator/random/np_uniform_op.cc
new file mode 100644
index 000000000000..d93991f63777
--- /dev/null
+++ b/src/api/operator/random/np_uniform_op.cc
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_uniform_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_uniform_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include <vector>
+#include "../utils.h"
+#include "../../../operator/numpy/random/np_uniform_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.uniform")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_uniform");
+  nnvm::NodeAttrs attrs;
+  op::NumpyUniformParam param;
+  int num_inputs = 0;
+  std::vector<NDArray*> inputs;
+  if (args[0].type_code() == kDLFloat || args[0].type_code() == kDLInt) {
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // 'low' and 'high' are both numeric types
+      num_inputs = 0;
+      param.low = args[0].operator double();
+      param.high = args[1].operator double();
+    } else {
+      // 'low' is numeric types but 'high' is not numeric types
+      num_inputs = 1;
+      param.low = args[0].operator double();
+      param.high = dmlc::nullopt;
+    }
+  } else {
+    if (args[1].type_code() == kDLFloat || args[1].type_code() == kDLInt) {
+      // 'low' is not numeric types but 'high' is numeric types
+      num_inputs = 1;
+      param.low = dmlc::nullopt;
+      param.high = args[1].operator double();
+    } else {
+      // nither 'low' or 'high' is numeric types
+      num_inputs = 2;
+    }
+  }
+  for (int i = 0; i < num_inputs; ++i) {
+    inputs.push_back(args[i].operator mxnet::NDArray*());
+  }
+  if (args[2].type_code() == kNull) {
+    param.size = dmlc::optional<mxnet::Tuple<int>>();
+  } else if (args[2].type_code() == kDLInt ||
+             args[2].type_code() == kDLFloat) {
+    param.size = Tuple<int>(1, args[2].operator int64_t());
+  } else {
+    param.size = Tuple<int>(args[2].operator ObjectRef());
+  }
+  if (args[4].type_code() == kNull) {
+    param.dtype = mshadow::kFloat32;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[4].operator std::string());
+  }
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  if (args[3].type_code() != kNull) {
+    attrs.dict["ctx"] = args[3].operator std::string();
+  }
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  SetAttrDict<op::NumpyUniformParam>(&attrs);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs.data(), &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_broadcast_reduce_op.h b/src/operator/numpy/np_broadcast_reduce_op.h
index 52ce81b43bcc..33cee78ebf80 100644
--- a/src/operator/numpy/np_broadcast_reduce_op.h
+++ b/src/operator/numpy/np_broadcast_reduce_op.h
@@ -31,6 +31,7 @@
 #include "../nn/moments-inl.h"
 #include "../tensor/broadcast_reduce_op.h"
 #include "../tensor/elemwise_binary_broadcast_op.h"
+#include "../../api/operator/op_utils.h"
 
 namespace mxnet {
 namespace op {
@@ -66,6 +67,21 @@ struct NumpyReduceAxesParam : public dmlc::Parameter<NumpyReduceAxesParam> {
     DMLC_DECLARE_FIELD(initial).set_default(dmlc::optional<double>())
       .describe("Starting value for the sum.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, dtype_s, keepdims_s, initial_s;
+    axis_s << axis;
+    dtype_s << dtype;
+    keepdims_s << keepdims;
+    initial_s << initial;
+    (*dict)["axis"] = axis_s.str();
+    if (dtype.has_value()) {
+      (*dict)["dtype"] = MXNetTypeWithBool2String(dtype.value());
+    } else {
+      (*dict)["dtype"] = dtype_s.str();
+    }
+    (*dict)["keepdims"] = keepdims_s.str();
+    (*dict)["initial"] = initial_s.str();
+  }
 };
 
 struct NumpyReduceAxesNoDTypeParam : public dmlc::Parameter<NumpyReduceAxesNoDTypeParam> {
diff --git a/src/operator/numpy/np_delete_op-inl.h b/src/operator/numpy/np_delete_op-inl.h
index b759afe892de..f39da5723c19 100644
--- a/src/operator/numpy/np_delete_op-inl.h
+++ b/src/operator/numpy/np_delete_op-inl.h
@@ -25,6 +25,7 @@
 #ifndef MXNET_OPERATOR_NUMPY_NP_DELETE_OP_INL_H_
 #define MXNET_OPERATOR_NUMPY_NP_DELETE_OP_INL_H_
 
+#include <string>
 #include <vector>
 #include <memory>
 #include <algorithm>
@@ -70,6 +71,19 @@ struct NumpyDeleteParam : public dmlc::Parameter<NumpyDeleteParam> {
     .set_default(dmlc::optional<int>())
     .describe("Axis along which to insert `values`.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream start_s, stop_s, step_s, int_ind_s, axis_s;
+    start_s << start;
+    stop_s << stop;
+    step_s << step;
+    int_ind_s << int_ind;
+    axis_s << axis;
+    (*dict)["start"] = start_s.str();
+    (*dict)["stop"] = stop_s.str();
+    (*dict)["step"] = step_s.str();
+    (*dict)["int_ind"] = int_ind_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 namespace delete_ {
diff --git a/src/operator/numpy/np_init_op.h b/src/operator/numpy/np_init_op.h
index 021989eb5fce..1288cf9e6225 100644
--- a/src/operator/numpy/np_init_op.h
+++ b/src/operator/numpy/np_init_op.h
@@ -29,6 +29,7 @@
 #include <algorithm>
 #include <vector>
 #include <string>
+#include "../../api/operator/op_utils.h"
 #include "../tensor/init_op.h"
 #include "../tensor/elemwise_unary_op.h"
 #include "../../api/operator/op_utils.h"
@@ -63,6 +64,17 @@ struct NumpyEyeParam : public dmlc::Parameter<NumpyEyeParam> {
     MXNET_ADD_ALL_TYPES
     .describe("Data-type of the returned array.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream N_s, M_s, k_s, dtype_s;
+    N_s << N;
+    M_s << M;
+    k_s << k;
+    dtype_s << dtype;
+    (*dict)["N"] = N_s.str();
+    (*dict)["M"] = M_s.str();
+    (*dict)["k"] = k_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 struct IndicesOpParam : public dmlc::Parameter<IndicesOpParam> {
@@ -251,6 +263,20 @@ struct LogspaceParam : public dmlc::Parameter<LogspaceParam> {
     MXNET_ADD_ALL_TYPES
     .describe("Target data type.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream start_s, stop_s, num_s, endpoint_s, base_s, dtype_s;
+    start_s << start;
+    stop_s << stop;
+    num_s << num;
+    endpoint_s << endpoint;
+    base_s << base;
+    (*dict)["start"] = start_s.str();
+    (*dict)["stop"] = stop_s.str();
+    (*dict)["num"] = num_s.str();
+    (*dict)["endpoint"] = endpoint_s.str();
+    (*dict)["base"] = base_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 struct logspace_fwd {
diff --git a/src/operator/numpy/np_insert_op-inl.h b/src/operator/numpy/np_insert_op-inl.h
index 6e7c431807cd..37591f0d3f8e 100644
--- a/src/operator/numpy/np_insert_op-inl.h
+++ b/src/operator/numpy/np_insert_op-inl.h
@@ -25,6 +25,7 @@
 #ifndef MXNET_OPERATOR_NUMPY_NP_INSERT_OP_INL_H_
 #define MXNET_OPERATOR_NUMPY_NP_INSERT_OP_INL_H_
 
+#include <string>
 #include <vector>
 #include <memory>
 #include <algorithm>
@@ -65,6 +66,21 @@ struct NumpyInsertParam : public dmlc::Parameter<NumpyInsertParam> {
     .set_default(dmlc::optional<int>())
     .describe("Axis along which to insert 'values'.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream val_s, start_s, stop_s, step_s, int_ind_s, axis_s;
+    val_s << val;
+    start_s << start;
+    stop_s << stop;
+    step_s << step;
+    int_ind_s << int_ind;
+    axis_s << axis;
+    (*dict)["val"] = val_s.str();
+    (*dict)["start"] = start_s.str();
+    (*dict)["stop"] = stop_s.str();
+    (*dict)["step"] = step_s.str();
+    (*dict)["int_ind"] = int_ind_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 /*!
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index ed6bf27ac0f9..223431198c82 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -1005,6 +1005,13 @@ struct NumpyConcatenateParam : public dmlc::Parameter<NumpyConcatenateParam> {
     .describe("The axis along which `values` are appended.  If `axis` is not"
               "given, both `arr` and `values` are flattened before use.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream num_args_s, axis_s;
+    num_args_s << num_args;
+    axis_s << axis;
+    (*dict)["num_args"] = num_args_s.str();
+    (*dict)["axis"] = axis_s.str();
+  }
 };
 
 template<typename xpu>
diff --git a/src/operator/numpy/np_window_op.h b/src/operator/numpy/np_window_op.h
index 7501c7643b8e..be85f19b3371 100644
--- a/src/operator/numpy/np_window_op.h
+++ b/src/operator/numpy/np_window_op.h
@@ -28,6 +28,7 @@
 
 #include <string>
 #include <vector>
+#include "../../api/operator/op_utils.h"
 #include "../tensor/init_op.h"
 
 namespace mxnet {
@@ -58,6 +59,13 @@ struct NumpyWindowsParam : public dmlc::Parameter<NumpyWindowsParam> {
     MXNET_ADD_ALL_TYPES
     .describe("Data-type of the returned array.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream M_s, dtype_s;
+    M_s << M;
+    dtype_s << dtype;
+    (*dict)["M"] = M_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 struct hanning_fwd {
diff --git a/src/operator/numpy/random/np_gamma_op.h b/src/operator/numpy/random/np_gamma_op.h
index 7c472060e1aa..c87ca655e551 100644
--- a/src/operator/numpy/random/np_gamma_op.h
+++ b/src/operator/numpy/random/np_gamma_op.h
@@ -31,6 +31,7 @@
 #include <string>
 #include <algorithm>
 #include "./dist_common.h"
+#include "../../../api/operator/op_utils.h"
 #include "../../elemwise_op_common.h"
 #include "../../tensor/elemwise_binary_broadcast_op.h"
 #include "../../mshadow_op.h"
@@ -68,6 +69,17 @@ struct NumpyGammaParam : public dmlc::Parameter<NumpyGammaParam> {
       .describe("DType of the output in case this can't be inferred. "
                 "Defaults to float32 if not defined (dtype=None).");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream shape_s, scale_s, dtype_s, size_s;
+    shape_s << shape;
+    scale_s << scale;
+    dtype_s << dtype;
+    size_s << size;
+    (*dict)["shape"] = shape_s.str();
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 
diff --git a/src/operator/numpy/random/np_normal_op.h b/src/operator/numpy/random/np_normal_op.h
index 8cc42887e5f4..4dd44060f49b 100644
--- a/src/operator/numpy/random/np_normal_op.h
+++ b/src/operator/numpy/random/np_normal_op.h
@@ -30,6 +30,7 @@
 #include <algorithm>
 #include <string>
 #include <vector>
+#include "../../../api/operator/op_utils.h"
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -67,6 +68,17 @@ struct NumpyNormalParam : public dmlc::Parameter<NumpyNormalParam> {
             "DType of the output in case this can't be inferred. "
             "Defaults to float32 if not defined (dtype=None).");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream loc_s, scale_s, dtype_s, size_s;
+    loc_s << loc;
+    scale_s << scale;
+    dtype_s << dtype;
+    size_s << size;
+    (*dict)["loc"] = loc_s.str();
+    (*dict)["scale"] = scale_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 inline bool NumpyNormalOpType(const nnvm::NodeAttrs &attrs,
diff --git a/src/operator/numpy/random/np_uniform_op.h b/src/operator/numpy/random/np_uniform_op.h
index 1df0c39d4e57..06f2aeec0b6c 100644
--- a/src/operator/numpy/random/np_uniform_op.h
+++ b/src/operator/numpy/random/np_uniform_op.h
@@ -29,6 +29,7 @@
 #include <algorithm>
 #include <string>
 #include <vector>
+#include "../../../api/operator/op_utils.h"
 #include "../../elemwise_op_common.h"
 #include "../../mshadow_op.h"
 #include "../../mxnet_op.h"
@@ -66,6 +67,17 @@ struct NumpyUniformParam : public dmlc::Parameter<NumpyUniformParam> {
             "DType of the output in case this can't be inferred. "
             "Defaults to float32 if not defined (dtype=None).");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream low_s, high_s, dtype_s, size_s;
+    low_s << low;
+    high_s << high;
+    dtype_s << dtype;
+    size_s << size;
+    (*dict)["low"] = low_s.str();
+    (*dict)["high"] = high_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+    (*dict)["size"] = size_s.str();
+  }
 };
 
 inline bool NumpyUniformOpType(const nnvm::NodeAttrs &attrs,
diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h
index ed0569c799ce..b78ed00622ef 100644
--- a/src/operator/tensor/init_op.h
+++ b/src/operator/tensor/init_op.h
@@ -35,6 +35,7 @@
 #include <string>
 #include <algorithm>
 #include <limits>
+#include "../../api/operator/op_utils.h"
 #include "../mshadow_op.h"
 #include "../elemwise_op_common.h"
 #include "../mxnet_op.h"
@@ -226,6 +227,21 @@ struct RangeParam : public dmlc::Parameter<RangeParam> {
     MXNET_ADD_ALL_TYPES
     .describe("Target data type.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream start_s, stop_s, step_s, repeat_s, infer_range_s, dtype_s;
+    start_s << start;
+    stop_s << stop;
+    step_s << step;
+    repeat_s << repeat;
+    infer_range_s << infer_range;
+    dtype_s << dtype;
+    (*dict)["start"] = start_s.str();
+    (*dict)["stop"] = stop_s.str();
+    (*dict)["step"] = step_s.str();
+    (*dict)["repeat"] = repeat_s.str();
+    (*dict)["infer_range"] = infer_range_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 struct RangeLikeParam : public dmlc::Parameter<RangeLikeParam> {
@@ -316,6 +332,19 @@ struct LinspaceParam : public dmlc::Parameter<LinspaceParam> {
     MXNET_ADD_ALL_TYPES
     .describe("Target data type.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream start_s, stop_s, num_s, endpoint_s, dtype_s;
+    start_s << start;
+    stop_s << stop;
+    num_s << num;
+    endpoint_s << endpoint;
+    dtype_s << dtype;
+    (*dict)["start"] = start_s.str();
+    (*dict)["stop"] = stop_s.str();
+    (*dict)["num"] = num_s.str();
+    (*dict)["endpoint"] = endpoint_s.str();
+    (*dict)["dtype"] = MXNetTypeWithBool2String(dtype);
+  }
 };
 
 template<typename ParamType>

From be6623e243531a18392988d40579159fcc6f4465 Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Fri, 10 Apr 2020 15:00:55 +0800
Subject: [PATCH 32/44] * impl debug - FFI for linalg multioutput op (#17879)

* impl - benchmark

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py       |  4 ++
 python/mxnet/ndarray/numpy/linalg.py        |  8 +--
 src/api/operator/numpy/linalg/np_det.cc     | 43 ++++++++++++++
 src/api/operator/numpy/linalg/np_eig.cc     | 64 +++++++++++++++++++++
 src/api/operator/numpy/linalg/np_slogdet.cc | 44 ++++++++++++++
 src/operator/numpy/linalg/np_eig-inl.h      |  6 ++
 tests/python/unittest/test_numpy_op.py      |  3 +-
 7 files changed, 166 insertions(+), 6 deletions(-)
 create mode 100644 src/api/operator/numpy/linalg/np_det.cc
 create mode 100644 src/api/operator/numpy/linalg/np_eig.cc
 create mode 100644 src/api/operator/numpy/linalg/np_slogdet.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 2ab3226d967e..2792c38d7994 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -60,6 +60,10 @@ def prepare_workloads():
     OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("linalg.eig", pool['3x3'])
+    OpArgMngr.add_workload("linalg.eigh", pool['3x3'])
+    OpArgMngr.add_workload("linalg.det", pool['3x3'])
+    OpArgMngr.add_workload("linalg.slogdet", pool['3x3'])
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("linalg.cholesky", pool['1x1'])
     OpArgMngr.add_workload("linalg.qr", pool['3x3'])
diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index 54a01b60bd55..ea294e9369fb 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -607,7 +607,7 @@ def det(a):
     >>> np.linalg.det(a)
     array([-2., -3., -8.])
     """
-    return _npi.det(a)
+    return _api_internal.det(a)
 
 
 def slogdet(a):
@@ -673,7 +673,7 @@ def slogdet(a):
     >>> np.linalg.slogdet(np.eye(500) * 0.1)
     (1., -1151.2925464970228)
     """
-    return _npi.slogdet(a)
+    return tuple(_api_internal.slogdet(a))
 
 
 def solve(a, b):
@@ -1025,7 +1025,7 @@ def eig(a):
            [ 0.13086087, -0.04077047, -0.9325615 ],
            [ 0.4021404 , -0.29585576,  0.26117516]])
     """
-    w, v = _npi.eig(a)
+    w, v = _api_internal.eig(a)
     return (w, v)
 
 
@@ -1093,5 +1093,5 @@ def eigh(a, UPLO='L'):
            [ 0.8242942 ,  0.56326365, -0.05721384],
            [-0.53661287,  0.80949366,  0.23825769]])
     """
-    w, v = _npi.eigh(a, UPLO)
+    w, v = _api_internal.eigh(a, UPLO)
     return (w, v)
diff --git a/src/api/operator/numpy/linalg/np_det.cc b/src/api/operator/numpy/linalg/np_det.cc
new file mode 100644
index 000000000000..2a415d1d56b5
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_det.cc
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_det.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/la_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.det")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_det");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = NDArrayHandle(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_eig.cc b/src/api/operator/numpy/linalg/np_eig.cc
new file mode 100644
index 000000000000..69f92a4762a1
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_eig.cc
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_eig.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_eig.cc
+ */
+
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_eig-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.eig")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_eig");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1])});
+});
+
+MXNET_REGISTER_API("_npi.eigh")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_eigh");
+  nnvm::NodeAttrs attrs;
+  op::EighParam param;
+  param.UPLO = *((args[1].operator std::string()).c_str());
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::EighParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1])});
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/linalg/np_slogdet.cc b/src/api/operator/numpy/linalg/np_slogdet.cc
new file mode 100644
index 000000000000..28c90265cdc7
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_slogdet.cc
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_slogdet.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/la_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.slogdet")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_slogdet");
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1])});
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_eig-inl.h b/src/operator/numpy/linalg/np_eig-inl.h
index a6ea19da2f4d..da7b99cb7611 100644
--- a/src/operator/numpy/linalg/np_eig-inl.h
+++ b/src/operator/numpy/linalg/np_eig-inl.h
@@ -26,6 +26,7 @@
 #define MXNET_OPERATOR_NUMPY_LINALG_NP_EIG_INL_H_
 
 #include <vector>
+#include <string>
 #include "./np_eigvals-inl.h"
 
 namespace mxnet {
@@ -216,6 +217,11 @@ struct EighParam : public dmlc::Parameter<EighParam> {
     .set_default('L')
     .describe("Specifies whether the calculation is done with the lower or upper triangular part.");
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream UPLO_s;
+    UPLO_s << UPLO;
+    (*dict)["UPLO"] = UPLO_s.str();
+  }
 };
 
 template<typename xpu>
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 06cbb355d130..29ebd00a25ff 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -6198,8 +6198,7 @@ def hybrid_forward(self, F, a):
         a_sym = mx.sym.Variable("a").as_np_ndarray()
         mx_sym = mx.sym.np.linalg.det(a_sym).as_nd_ndarray()
         if 0 not in shape and grad_req != 'null':
-            check_numeric_gradient(mx_sym, [a.as_nd_ndarray()],
-                                    rtol=1e-1, atol=1e-1, dtype=dtype)
+            check_numeric_gradient(mx_sym, [a.as_nd_ndarray()], rtol=1e-1, atol=1e-1, dtype=dtype)
 
 
 @with_seed()

From ff2dbab51f130f7f93a705724f77585aa3e39288 Mon Sep 17 00:00:00 2001
From: dw_sjtu <46704444+sjtuWangDing@users.noreply.github.com>
Date: Tue, 14 Apr 2020 12:57:58 +0800
Subject: [PATCH 33/44] * impl - linalg matrix_rank for cpu and gpu implemented
 (#18020)

* fix - python interface

* impl - ffi for matrix_rank

* impl - ffi benchmark

Co-authored-by: Ubuntu <ubuntu@ip-172-31-10-214.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py         |   9 +-
 python/mxnet/ndarray/numpy/linalg.py          |  47 +-
 python/mxnet/numpy/fallback_linalg.py         |   2 -
 python/mxnet/numpy/linalg.py                  |  43 +-
 python/mxnet/numpy_dispatch_protocol.py       |   1 +
 python/mxnet/symbol/numpy/linalg.py           |  35 +-
 .../operator/numpy/linalg/np_matrix_rank.cc   |  76 +++
 .../numpy/linalg/np_matrix_rank-inl.h         | 449 ++++++++++++++++++
 src/operator/numpy/linalg/np_matrix_rank.cc   | 165 +++++++
 src/operator/numpy/linalg/np_matrix_rank.cu   |  38 ++
 .../unittest/test_numpy_interoperability.py   |  29 +-
 tests/python/unittest/test_numpy_op.py        |  77 +++
 12 files changed, 956 insertions(+), 15 deletions(-)
 create mode 100644 src/api/operator/numpy/linalg/np_matrix_rank.cc
 create mode 100644 src/operator/numpy/linalg/np_matrix_rank-inl.h
 create mode 100644 src/operator/numpy/linalg/np_matrix_rank.cc
 create mode 100644 src/operator/numpy/linalg/np_matrix_rank.cu

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 2792c38d7994..2711cf9d06e3 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -64,6 +64,7 @@ def prepare_workloads():
     OpArgMngr.add_workload("linalg.eigh", pool['3x3'])
     OpArgMngr.add_workload("linalg.det", pool['3x3'])
     OpArgMngr.add_workload("linalg.slogdet", pool['3x3'])
+    OpArgMngr.add_workload("linalg.matrix_rank", pool['3x3'], pool['1'], hermitian=False)
     OpArgMngr.add_workload("linalg.svd", pool['3x3'])
     OpArgMngr.add_workload("linalg.cholesky", pool['1x1'])
     OpArgMngr.add_workload("linalg.qr", pool['3x3'])
@@ -123,10 +124,10 @@ def prepare_workloads():
                            out=dnp.array([False, False], dtype=bool), keepdims=False)
     OpArgMngr.add_workload("roll", pool["2x2"], 1, axis=0)
     OpArgMngr.add_workload("rot90", pool["2x2"], 2)
-    OpArgMngr.add_workload("array_split", pool['2X2'], 2, axis=1)
-    OpArgMngr.add_workload("vsplit", pool['2X2'], 2)
-    OpArgMngr.add_workload("hsplit", pool['2X2'], 2)
-    OpArgMngr.add_workload("dsplit", pool['2X2x2'], 2)
+    OpArgMngr.add_workload("array_split", pool['2x2'], 2, axis=1)
+    OpArgMngr.add_workload("vsplit", pool['2x2'], 2)
+    OpArgMngr.add_workload("hsplit", pool['2x2'], 2)
+    OpArgMngr.add_workload("dsplit", pool['2x2x2'], 2)
     OpArgMngr.add_workload("arange", 10)
     OpArgMngr.add_workload("concatenate", (pool['1x2'], pool['1x2'], pool['1x2']), axis=0)
     OpArgMngr.add_workload("append", pool['2x2'], pool['1x2'], axis=0)
diff --git a/python/mxnet/ndarray/numpy/linalg.py b/python/mxnet/ndarray/numpy/linalg.py
index ea294e9369fb..d31e8ea7921d 100644
--- a/python/mxnet/ndarray/numpy/linalg.py
+++ b/python/mxnet/ndarray/numpy/linalg.py
@@ -23,7 +23,52 @@
 from . import _api_internal
 
 __all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
-           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank']
+
+
+def matrix_rank(M, tol=None, hermitian=False):
+    """
+    Return matrix rank of array using SVD method
+
+    Rank of the array is the number of singular values of the array that are
+    greater than `tol`.
+
+    Parameters
+    M : {(M,), (..., M, N)} ndarray
+        Input vector or stack of matrices.
+    tol : (...) ndarray, float, optional
+        Threshold below which SVD values are considered zero. If `tol` is
+        None, and ``S`` is an array with singular values for `M`, and
+        ``eps`` is the epsilon value for datatype of ``S``, then `tol` is
+        set to ``S.max() * max(M.shape) * eps``.
+    hermitian : bool, optional
+        If True, `M` is assumed to be Hermitian (symmetric if real-valued),
+        enabling a more efficient method for finding singular values.
+        Defaults to False.
+
+    Returns
+    -------
+    rank : (...) ndarray
+        Rank of M.
+
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> np.matrix_rank(np.eye(4)) # Full rank matrix
+    4
+    >>> I=np.eye(4); I[-1,-1] = 0. # rank deficient matrix
+    >>> np.matrix_rank(I)
+    3
+    >>> np.matrix_rank(np.ones((4,))) # 1 dimension - rank 1 unless all 0
+    1
+    >>> np.matrix_rank(np.zeros((4,)))
+    0
+    """
+    finfo_eps_32 = _np.finfo(_np.float32).eps
+    finfo_eps_64 = _np.finfo(_np.float64).eps
+    if hermitian is True:
+        raise NotImplementedError("hermitian is not supported yet...")
+    return _api_internal.matrix_rank(M, tol, hermitian, finfo_eps_32, finfo_eps_64)
 
 
 def lstsq(a, b, rcond='warn'):
diff --git a/python/mxnet/numpy/fallback_linalg.py b/python/mxnet/numpy/fallback_linalg.py
index 5e06ff94a4ce..79d6b83062ec 100644
--- a/python/mxnet/numpy/fallback_linalg.py
+++ b/python/mxnet/numpy/fallback_linalg.py
@@ -24,11 +24,9 @@
 __all__ = [
     'cond',
     'matrix_power',
-    'matrix_rank',
     'multi_dot'
 ]
 
 cond = onp.linalg.cond
 matrix_power = onp.linalg.matrix_power
-matrix_rank = onp.linalg.matrix_rank
 multi_dot = onp.linalg.multi_dot
diff --git a/python/mxnet/numpy/linalg.py b/python/mxnet/numpy/linalg.py
index 445adfdeaeae..d2756d531b7c 100644
--- a/python/mxnet/numpy/linalg.py
+++ b/python/mxnet/numpy/linalg.py
@@ -22,10 +22,51 @@
 from . import fallback_linalg
 
 __all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
-           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank']
 __all__ += fallback_linalg.__all__
 
 
+def matrix_rank(M, tol=None, hermitian=False):
+    """
+    Return matrix rank of array using SVD method
+
+    Rank of the array is the number of singular values of the array that are
+    greater than `tol`.
+
+    Parameters
+    M : {(M,), (..., M, N)} ndarray
+        Input vector or stack of matrices.
+    tol : (...) ndarray, float, optional
+        Threshold below which SVD values are considered zero. If `tol` is
+        None, and ``S`` is an array with singular values for `M`, and
+        ``eps`` is the epsilon value for datatype of ``S``, then `tol` is
+        set to ``S.max() * max(M.shape) * eps``.
+    hermitian : bool, optional
+        If True, `M` is assumed to be Hermitian (symmetric if real-valued),
+        enabling a more efficient method for finding singular values.
+        Defaults to False.
+
+    Returns
+    -------
+    rank : (...) ndarray
+        Rank of M.
+
+    Examples
+    --------
+    >>> from mxnet import np
+    >>> np.matrix_rank(np.eye(4)) # Full rank matrix
+    4
+    >>> I=np.eye(4); I[-1,-1] = 0. # rank deficient matrix
+    >>> np.matrix_rank(I)
+    3
+    >>> np.matrix_rank(np.ones((4,))) # 1 dimension - rank 1 unless all 0
+    1
+    >>> np.matrix_rank(np.zeros((4,)))
+    0
+    """
+    return _mx_nd_np.linalg.matrix_rank(M, tol, hermitian)
+
+
 def lstsq(a, b, rcond='warn'):
     r"""
     Return the least-squares solution to a linear matrix equation.
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index f25bddf8662d..9c955e5702b5 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -165,6 +165,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'linalg.eigvalsh',
     'linalg.eigh',
     'linalg.qr',
+    'linalg.matrix_rank',
     'shape',
     'trace',
     'tril',
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index c16aef00e48d..3cea6ddae157 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -23,7 +23,40 @@
 from . import _internal as _npi
 
 __all__ = ['norm', 'svd', 'cholesky', 'qr', 'inv', 'det', 'slogdet', 'solve', 'tensorinv', 'tensorsolve',
-           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq']
+           'pinv', 'eigvals', 'eig', 'eigvalsh', 'eigh', 'lstsq', 'matrix_rank']
+
+
+def matrix_rank(M, tol=None, hermitian=False):
+    """
+    Return matrix rank of array using SVD method
+
+    Rank of the array is the number of singular values of the array that are
+    greater than `tol`.
+
+    Parameters
+    M : {(M,), (..., M, N)} _Symbol
+        Input vector or stack of matrices.
+    tol : (...) _Symbol, float, optional
+        Threshold below which SVD values are considered zero. If `tol` is
+        None, and ``S`` is an array with singular values for `M`, and
+        ``eps`` is the epsilon value for datatype of ``S``, then `tol` is
+        set to ``S.max() * max(M.shape) * eps``.
+    hermitian : bool, optional
+        If True, `M` is assumed to be Hermitian (symmetric if real-valued),
+        enabling a more efficient method for finding singular values.
+        Defaults to False.
+
+    Returns
+    -------
+    rank : (...) _Symbol
+        Rank of M.
+    """
+    finfo_eps_32 = _np.finfo(_np.float32).eps
+    finfo_eps_64 = _np.finfo(_np.float64).eps
+    if tol is None:
+        return _npi.matrix_rank_none_tol(M, finfo_eps_32, finfo_eps_64, hermitian)
+    else:
+        return _npi.matrix_rank(M, tol, hermitian)
 
 
 def lstsq(a, b, rcond='warn'):
diff --git a/src/api/operator/numpy/linalg/np_matrix_rank.cc b/src/api/operator/numpy/linalg/np_matrix_rank.cc
new file mode 100644
index 000000000000..4bfe66664ef8
--- /dev/null
+++ b/src/api/operator/numpy/linalg/np_matrix_rank.cc
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_pinv.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/linalg/np_matrix_rank.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/linalg/np_matrix_rank-inl.h"
+
+namespace mxnet {
+
+inline static void _npi_matrix_rank_none_tol(runtime::MXNetArgs args,
+                                             runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_matrix_rank_none_tol");
+  op::MatrixRankNoneTolParam param;
+  nnvm::NodeAttrs attrs;
+  param.hermitian = args[2].operator bool();
+  param.finfoEps32 = args[3].operator double();
+  param.finfoEps64 = args[4].operator double();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::MatrixRankNoneTolParam>(&attrs);
+  int num_inputs = 1;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+inline static void _npi_matrix_rank(runtime::MXNetArgs args,
+                                    runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_matrix_rank");
+  op::MatrixRankParam param;
+  nnvm::NodeAttrs attrs;
+  param.hermitian = args[2].operator bool();
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::MatrixRankParam>(&attrs);
+  int num_inputs = 2;
+  int num_outputs = 0;
+  NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+}
+
+MXNET_REGISTER_API("_npi.matrix_rank")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  if (args[1].type_code() == kNull) {
+    _npi_matrix_rank_none_tol(args, ret);
+  } else {
+    _npi_matrix_rank(args, ret);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_matrix_rank-inl.h b/src/operator/numpy/linalg/np_matrix_rank-inl.h
new file mode 100644
index 000000000000..8ccecb57db11
--- /dev/null
+++ b/src/operator/numpy/linalg/np_matrix_rank-inl.h
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_matrix_rank-inl.h
+ * \brief Placeholder for matrix_rank
+ */
+#ifndef MXNET_OPERATOR_NUMPY_LINALG_NP_MATRIX_RANK_INL_H_
+#define MXNET_OPERATOR_NUMPY_LINALG_NP_MATRIX_RANK_INL_H_
+
+#include <mxnet/operator_util.h>
+#include <vector>
+#include <string>
+#include <utility>
+#include <algorithm>
+#include "../../operator_common.h"
+#include "../../mshadow_op.h"
+#include "./np_pinv-inl.h"
+
+namespace mxnet {
+namespace op {
+
+using namespace mshadow;
+
+struct MatrixRankNoneTolParam : public dmlc::Parameter<MatrixRankNoneTolParam> {
+  float finfoEps32;
+  double finfoEps64;
+  bool hermitian;
+  DMLC_DECLARE_PARAMETER(MatrixRankNoneTolParam) {
+    DMLC_DECLARE_FIELD(finfoEps32)
+    .set_default(0)
+    .describe("Machine limits for float32 type");
+    DMLC_DECLARE_FIELD(finfoEps64)
+    .set_default(0)
+    .describe("Machine limits for float64 type");
+    DMLC_DECLARE_FIELD(hermitian)
+    .set_default(false)
+    .describe("If True, M is assumed to be Hermitian (symmetric if real-valued).");
+  }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream finfoEps32_s, finfoEps64_s, hermitian_s;
+    finfoEps32_s << finfoEps32;
+    finfoEps64_s << finfoEps64;
+    hermitian_s << hermitian;
+    (*dict)["finfoEps32"] = finfoEps32_s.str();
+    (*dict)["finfoEps64"] = finfoEps64_s.str();
+    (*dict)["hermitian"] = hermitian_s.str();
+  }
+};
+
+struct MatrixRankParam : public dmlc::Parameter<MatrixRankParam> {
+  bool hermitian;
+  DMLC_DECLARE_PARAMETER(MatrixRankParam) {
+    DMLC_DECLARE_FIELD(hermitian)
+    .set_default(false)
+    .describe("If True, M is assumed to be Hermitian (symmetric if real-valued).");
+  }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream hermitian_s;
+    hermitian_s << hermitian;
+    (*dict)["hermitian"] = hermitian_s.str();
+  }
+};
+
+template<int req>
+struct VectorRankKernel {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *in_data,
+                                  int64_t *out_data, const int& data_size) {
+    bool all_nozero = true;
+    for (int j = 0; j < data_size; ++j) {
+      if (!((in_data[j] > 0 ? in_data[j] : -in_data[j]) > 0)) {
+        all_nozero = false;
+        break;
+      }
+    }
+    KERNEL_ASSIGN(*out_data, req, static_cast<int64_t>(all_nozero ? 1 : 0));
+  }
+};
+
+template<int req>
+struct MatrixRankNoneTolKernel {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *in_data, int64_t *out_data,
+                                  const int& nrow, const int& ncol, const double& finfoEps,
+                                  const int& data_size, const int& batch_size) {
+    if (i < batch_size) {
+      DType max_singular_value = 0;
+      for (int j = 0; j < data_size; ++j) {
+        DType sv = in_data[j + i * data_size];
+        max_singular_value = sv > max_singular_value ? sv : max_singular_value;
+      }
+      double tol = (nrow > ncol ? nrow : ncol) * static_cast<double>(max_singular_value) * finfoEps;
+      int64_t rank_num = 0;
+      for (int j = 0; j < data_size; ++j) {
+        rank_num += in_data[j + i * data_size] > tol ? 1 : 0;
+      }
+      KERNEL_ASSIGN(out_data[i], req, rank_num);
+    }
+  }
+};
+
+template<int req>
+struct MatrixRankKernel {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, const DType *in_data, int64_t *out_data,
+                                  const int& data_size, const int& batch_size) {
+    if (i < batch_size) {
+      int64_t rank_num = 0;
+      for (int j = 0; j < data_size; ++j) {
+        rank_num += in_data[j + i * data_size] > 0 ? 1 : 0;
+      }
+      KERNEL_ASSIGN(out_data[i], req, rank_num);
+    }
+  }
+};
+
+struct SVDWrapper {
+  template<typename xpu, typename DType>
+  static void op(const TBlob& a, const TBlob& s,
+                 const TBlob& u, const mxnet::TShape& ut_shape,
+                 const TBlob& v, const mxnet::TShape& vt_shape,
+                 const TBlob& work, const OpContext& ctx) {
+    Stream<xpu> *s_xpu = ctx.get_stream<xpu>();
+    const mxnet::TShape& a_shape = a.shape_;
+    const mxnet::TShape& ut_axis = GetTransAxis(u.shape_);
+    const int a_ndim = a.ndim();
+    const int nrow = a_shape[a_ndim - 2];
+    const int ncol = a_shape[a_ndim - 1];
+    if (nrow > ncol) {
+      const_cast<TBlob&>(u) = u.reshape(ut_shape);
+      const_cast<TBlob&>(v) = v.reshape(vt_shape);
+      mxnet::op::TransposeImpl<xpu>(ctx.run_ctx, a, u, ut_axis);
+      BatchSVDImpl(ncol, nrow,
+                   v.FlatToKD<xpu, 3, DType>(s_xpu),
+                   s.FlatToKD<xpu, 2, DType>(s_xpu),
+                   u.FlatToKD<xpu, 3, DType>(s_xpu),
+                   work.FlatToKD<xpu, 1, DType>(s_xpu), s_xpu);
+    } else {
+      if (a.dptr<DType>() != v.dptr<DType>()) {
+        Copy(v.FlatToKD<xpu, 3, DType>(s_xpu), a.FlatToKD<xpu, 3, DType>(s_xpu), s_xpu);
+      }
+      BatchSVDImpl(nrow, ncol,
+                   u.FlatToKD<xpu, 3, DType>(s_xpu),
+                   s.FlatToKD<xpu, 2, DType>(s_xpu),
+                   v.FlatToKD<xpu, 3, DType>(s_xpu),
+                   work.FlatToKD<xpu, 1, DType>(s_xpu), s_xpu);
+    }
+  }
+};
+
+inline void GetOrCheckBroadcastShape(const nnvm::NodeAttrs& attrs,
+                                     const mxnet::TShape& a_shape,
+                                     const mxnet::TShape& tol_shape,
+                                     mxnet::TShape *broadcast_shape = nullptr,
+                                     mxnet::TShape *new_tol_shape = nullptr) {
+  CHECK_GE(a_shape.ndim(), 2);
+  const int a_ndim = a_shape.ndim();
+  const int tol_ndim = tol_shape.ndim();
+  const int nrow = a_shape[a_ndim - 2];
+  const int ncol = a_shape[a_ndim - 1];
+  // Get new tol shape.
+  mxnet::TShape temp_new_tol_shape(tol_ndim + 1, 1);
+  for (int i = 0; i < tol_ndim; ++i) { temp_new_tol_shape[i] = tol_shape[i]; }
+  // Get singular value shape.
+  mxnet::TShape temp_s_shape(a_ndim - 1, 0);
+  for (int i = 0; i < a_ndim - 2; ++i) {
+    temp_s_shape[i] = a_shape[i];
+  }
+  temp_s_shape[a_ndim - 2] = std::min(nrow, ncol);
+  // Check binary broadcast shape.
+  mxnet::ShapeVector in_shape_vec({ temp_s_shape, temp_new_tol_shape });
+  mxnet::ShapeVector out_shape_vec(1, mxnet::TShape());
+  mxnet::op::BinaryBroadcastShape(attrs, &in_shape_vec, &out_shape_vec);
+  // Assign shape.
+  if (broadcast_shape) {
+    *broadcast_shape = out_shape_vec[0];
+  }
+  if (new_tol_shape) {
+    *new_tol_shape = temp_new_tol_shape;
+  }
+}
+
+template<typename xpu, typename DType>
+struct WSQ {
+  static size_t SVDWorkspaceSizeQuery(const TBlob& a,
+                                      const mxnet::TShape& u_shape,
+                                      const mxnet::TShape& s_shape,
+                                      const mxnet::TShape& v_shape,
+                                      const OpContext& ctx) {
+    size_t workspace_size = 0;
+    Stream<xpu> *s = ctx.get_stream<xpu>();
+    const int a_ndim = a.shape_.ndim();
+    const int u_ndim = u_shape.ndim();
+    const int s_ndim = s_shape.ndim();
+    const int v_ndim = v_shape.ndim();
+    mxnet::TShape u_shape2 = Shape2(u_shape[u_ndim - 2], u_shape[u_ndim - 1]);
+    mxnet::TShape s_shape1 = Shape1(s_shape[s_ndim - 1]);
+    mxnet::TShape v_shape2 = Shape2(v_shape[v_ndim - 2], v_shape[v_ndim - 1]);
+    if (xpu::kDevCPU) {
+      std::vector<DType> u_vec(u_shape2.Size(), 0);
+      std::vector<DType> s_vec(s_shape1.Size(), 0);
+      std::vector<DType> v_vec(v_shape2.Size(), 0);
+      // Get workspace size in linalg_gesdd.
+      workspace_size += linalg_gesdd_workspace_query(
+        a.shape_[a_ndim - 2], a.shape_[a_ndim - 1],
+        TBlob(u_vec.data(), u_shape2, a.dev_mask(), a.dev_id()).get<xpu, 2, DType>(s),
+        TBlob(s_vec.data(), s_shape1, a.dev_mask(), a.dev_id()).get<xpu, 1, DType>(s),
+        TBlob(v_vec.data(), v_shape2, a.dev_mask(), a.dev_id()).get<xpu, 2, DType>(s), s);
+    } else {
+      Storage::Handle u_handle =
+        Storage::Get()->Alloc(sizeof(DType) * u_shape2.Size(), Context::GPU());
+      Storage::Handle s_handle =
+        Storage::Get()->Alloc(sizeof(DType) * s_shape1.Size(), Context::GPU());
+      Storage::Handle v_handle =
+        Storage::Get()->Alloc(sizeof(DType) * v_shape2.Size(), Context::GPU());
+      TBlob u_data(static_cast<DType*>(u_handle.dptr), u_shape2, a.dev_mask(), a.dev_id());
+      TBlob s_data(static_cast<DType*>(s_handle.dptr), s_shape1, a.dev_mask(), a.dev_id());
+      TBlob v_data(static_cast<DType*>(v_handle.dptr), v_shape2, a.dev_mask(), a.dev_id());
+      // Get workspace size in linalg_gesvd.
+      if (a.shape_[a_ndim - 2] >= a.shape_[a_ndim - 1]) {
+        workspace_size += linalg_gesvd_workspace_query(v_data.get<xpu, 2, DType>(s),
+                                                       s_data.get<xpu, 1, DType>(s),
+                                                       u_data.get<xpu, 2, DType>(s), s);
+      } else {
+        workspace_size += linalg_gesvd_workspace_query(u_data.get<xpu, 2, DType>(s),
+                                                       s_data.get<xpu, 1, DType>(s),
+                                                       v_data.get<xpu, 2, DType>(s), s);
+      }
+      Storage::Get()->Free(u_handle);
+      Storage::Get()->Free(s_handle);
+      Storage::Get()->Free(v_handle);
+    }
+    return workspace_size;
+  }
+
+  static size_t MatrixRankNoneTolForwardWSQ(size_t *svd_workspace_size,
+                                            const TBlob& a,
+                                            const OpContext& ctx) {
+    size_t workspace_size = 0;
+    mxnet::TShape u_shape, s_shape, v_shape;
+    GetPinvShape(a.shape_, &u_shape, &s_shape, &v_shape);
+    *svd_workspace_size = SVDWorkspaceSizeQuery(a, u_shape, s_shape, v_shape, ctx);
+    workspace_size += *svd_workspace_size;  // For #gesdd_ or #gesvd work space.
+    workspace_size += u_shape.Size();       // For UT.
+    workspace_size += s_shape.Size();       // For S.
+    workspace_size += v_shape.Size();       // For V.
+    return workspace_size * sizeof(DType);
+  }
+
+  static size_t MatrixRankForwardWSQ(size_t *svd_workspace_size,
+                                     const TBlob& a,
+                                     const TBlob& tol,
+                                     const nnvm::NodeAttrs& attrs,
+                                     const OpContext& ctx) {
+    const mxnet::TShape a_shape = a.shape_;
+    const mxnet::TShape tol_shape = tol.shape_;
+    size_t workspace_size = 0;
+    mxnet::TShape u_shape, s_shape, v_shape;
+    GetPinvShape(a.shape_, &u_shape, &s_shape, &v_shape);
+    mxnet::TShape broadcast_shape, new_tol_shape;
+    GetOrCheckBroadcastShape(attrs, a_shape, tol_shape, &broadcast_shape, &new_tol_shape);
+    *svd_workspace_size = SVDWorkspaceSizeQuery(a, u_shape, s_shape, v_shape, ctx);
+    workspace_size += *svd_workspace_size;     // For #gesdd_ or #gesvd work space.
+    workspace_size += u_shape.Size();          // For UT.
+    workspace_size += s_shape.Size();          // For S.
+    workspace_size += v_shape.Size();          // For V.
+    workspace_size += new_tol_shape.Size();    // For tol with newaxis.
+    workspace_size += broadcast_shape.Size();  // For binary broadcast shape.
+    return workspace_size * sizeof(DType);
+  }
+};
+
+template<typename xpu>
+void MatrixRankNoneTolForwardImpl(const TBlob& a,
+                                  const TBlob& rank,
+                                  const nnvm::NodeAttrs& attrs,
+                                  const OpContext& ctx,
+                                  const std::vector<OpReqType>& req) {
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const mxnet::TShape& a_shape = a.shape_;
+  const int a_ndim = a.ndim();
+  MSHADOW_SGL_DBL_TYPE_SWITCH(a.type_flag_, DType, {
+    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+      if (a_ndim < 2) {
+        mxnet_op::Kernel<VectorRankKernel<req_type>, xpu>::Launch(
+          s, 1, a.dptr<DType>(), rank.dptr<int64_t>(), a.Size());
+        return;
+      }
+      // a_ndim >= 2
+      const int nrow = a_shape[a_ndim - 2];
+      const int ncol = a_shape[a_ndim - 1];
+      const MatrixRankNoneTolParam& param = nnvm::get<MatrixRankNoneTolParam>(attrs.parsed);
+      CHECK_EQ(param.hermitian, false)
+        << "matrix_rank not support param.hermitian = true at present.";
+      double finfoEps = a.type_flag_ == mshadow::kFloat32 ? param.finfoEps32 : param.finfoEps64;
+      // Step1: Calculate workspace size.
+      size_t svd_workspace_size = 0;
+      size_t workspace_size =
+        WSQ<xpu, DType>::MatrixRankNoneTolForwardWSQ(&svd_workspace_size, a, ctx);
+      Tensor<xpu, 1, char> workspace =
+        ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+      // Step2: Allocate memory.
+      mxnet::TShape s_shape, u_shape, v_shape, ut_shape, vt_shape;
+      GetPinvShape(a_shape, &u_shape, &s_shape, &v_shape, &ut_shape, &vt_shape);
+      DType *s_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+      DType *u_ptr = s_ptr + s_shape.Size();
+      DType *v_ptr = u_ptr + u_shape.Size();
+      DType *work_ptr = v_ptr + v_shape.Size();
+      TBlob s_data(s_ptr, s_shape, a.dev_mask(), a.dev_id());
+      TBlob u_data(u_ptr, u_shape, a.dev_mask(), a.dev_id());
+      TBlob v_data(v_ptr, v_shape, a.dev_mask(), a.dev_id());
+      TBlob work_data(work_ptr, Shape1(svd_workspace_size), a.dev_mask(), a.dev_id());
+      // Step3: SVD.
+      SVDWrapper::op<xpu, DType>(a, s_data, u_data, ut_shape, v_data, vt_shape, work_data, ctx);
+      // Step4: Calculate rank.
+      const int data_size = s_data.size(s_data.ndim() - 1);
+      const int batch_size = a_ndim == 2 ? 1 : s_shape.ProdShape(0, s_shape.ndim() - 1);
+      mxnet_op::Kernel<MatrixRankNoneTolKernel<req_type>, xpu>::Launch(s, batch_size,
+                                                                       s_data.dptr<DType>(),
+                                                                       rank.dptr<int64_t>(),
+                                                                       nrow, ncol, finfoEps,
+                                                                       data_size, batch_size);
+    });
+  });
+}
+
+template<typename xpu>
+void MatrixRankNoneTolForward(const nnvm::NodeAttrs& attrs,
+                              const OpContext& ctx,
+                              const std::vector<TBlob>& inputs,
+                              const std::vector<OpReqType>& req,
+                              const std::vector<TBlob>& outputs) {
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req.size(), 1U);
+  if (kNullOp == req[0]) { return; }
+  CHECK(req[0] == kWriteTo || req[0] == kWriteInplace);
+
+  const TBlob& a = inputs[0];
+  const TBlob& rank = outputs[0];
+  MatrixRankNoneTolForwardImpl<xpu>(a, rank, attrs, ctx, req);
+}
+
+template<typename xpu>
+void MatrixRankForwardImpl(const TBlob& a,
+                           const TBlob& tol,
+                           const TBlob& rank,
+                           const nnvm::NodeAttrs& attrs,
+                           const OpContext& ctx,
+                           const std::vector<OpReqType>& req) {
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const mxnet::TShape& a_shape = a.shape_;
+  const mxnet::TShape& tol_shape = tol.shape_;
+  const int a_ndim = a.ndim();
+  MSHADOW_SGL_DBL_TYPE_SWITCH(a.type_flag_, DType, {
+    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+      if (a_ndim < 2) {
+        mxnet_op::Kernel<VectorRankKernel<req_type>, xpu>::Launch(
+          s, 1, a.dptr<DType>(), rank.dptr<int64_t>(), a.Size());
+        return;
+      }
+      // a_ndim >= 2
+      const MatrixRankParam& param = nnvm::get<MatrixRankParam>(attrs.parsed);
+      CHECK_EQ(param.hermitian, false)
+        << "matrix_rank not support param.hermitian = true at present.";
+      mxnet::TShape s_shape, u_shape, v_shape, ut_shape, vt_shape;
+      GetPinvShape(a_shape, &u_shape, &s_shape, &v_shape, &ut_shape, &vt_shape);
+      mxnet::TShape broadcast_shape, new_tol_shape;
+      GetOrCheckBroadcastShape(attrs, a_shape, tol_shape, &broadcast_shape, &new_tol_shape);
+      // Step1: Calculate workspace size.
+      size_t svd_workspace_size = 0;
+      size_t workspace_size =
+        WSQ<xpu, DType>::MatrixRankForwardWSQ(&svd_workspace_size, a, tol, attrs, ctx);
+      Tensor<xpu, 1, char> workspace =
+        ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+      // Step2: Allocate memory.
+      DType *s_ptr = reinterpret_cast<DType*>(workspace.dptr_);
+      DType *u_ptr = s_ptr + s_shape.Size();
+      DType *v_ptr = u_ptr + u_shape.Size();
+      DType *work_ptr = v_ptr + v_shape.Size();
+      DType *new_tol_ptr = work_ptr + svd_workspace_size;
+      DType *broadcast_ptr = new_tol_ptr + new_tol_shape.Size();
+      TBlob s_data(s_ptr, s_shape, a.dev_mask(), a.dev_id());
+      TBlob u_data(u_ptr, u_shape, a.dev_mask(), a.dev_id());
+      TBlob v_data(v_ptr, v_shape, a.dev_mask(), a.dev_id());
+      TBlob work_data(work_ptr, Shape1(svd_workspace_size), a.dev_mask(), a.dev_id());
+      TBlob new_tol_data(new_tol_ptr, new_tol_shape, a.dev_mask(), a.dev_id());
+      TBlob broadcast_data(broadcast_ptr, broadcast_shape, a.dev_mask(), a.dev_id());
+      // Step3: SVD.
+      SVDWrapper::op<xpu, DType>(a, s_data, u_data, ut_shape, v_data, vt_shape, work_data, ctx);
+      // Step4: Calculate broadcast data.
+      if (new_tol_data.dptr<DType>() != tol.dptr<DType>()) {
+        Copy(new_tol_data.FlatTo1D<xpu, DType>(s), tol.FlatTo1D<xpu, DType>(s), s);
+      }
+      mxnet::op::BinaryBroadcastCompute<xpu, op::mshadow_op::gt>(attrs, ctx,
+                                                                 {s_data, new_tol_data},
+                                                                 {kWriteTo}, {broadcast_data});
+      // Step5: Calculate rank.
+      const int b_ndim  = broadcast_shape.ndim();
+      const int data_size = broadcast_data.size(b_ndim - 1);
+      const int batch_size = b_ndim == 1 ? 1 : broadcast_shape.ProdShape(0, b_ndim - 1);
+      mxnet_op::Kernel<MatrixRankKernel<req_type>, xpu>::Launch(s, batch_size,
+                                                                broadcast_data.dptr<DType>(),
+                                                                rank.dptr<int64_t>(),
+                                                                data_size, batch_size);
+    });
+  });
+}
+
+template<typename xpu>
+void MatrixRankForward(const nnvm::NodeAttrs& attrs,
+                       const OpContext& ctx,
+                       const std::vector<TBlob>& inputs,
+                       const std::vector<OpReqType>& req,
+                       const std::vector<TBlob>& outputs) {
+  CHECK_EQ(inputs.size(), 2U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req.size(), 1U);
+  if (kNullOp == req[0]) { return; }
+  CHECK(req[0] == kWriteTo || req[0] == kWriteInplace);
+
+  const TBlob& a = inputs[0];
+  const TBlob& tol = inputs[1];
+  const TBlob& rank = outputs[0];
+  MatrixRankForwardImpl<xpu>(a, tol, rank, attrs, ctx, req);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_LINALG_NP_MATRIX_RANK_INL_H_
diff --git a/src/operator/numpy/linalg/np_matrix_rank.cc b/src/operator/numpy/linalg/np_matrix_rank.cc
new file mode 100644
index 000000000000..d3794a1de0e9
--- /dev/null
+++ b/src/operator/numpy/linalg/np_matrix_rank.cc
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_matrix_rank.cc
+ * \brief CPU implementation of the matrix_rank Operator
+ */
+#include "./np_matrix_rank-inl.h"
+
+namespace mxnet {
+namespace op {
+
+inline bool MatrixRankNoneTolShape(const nnvm::NodeAttrs& attrs,
+                                   mxnet::ShapeVector *in_attrs,
+                                   mxnet::ShapeVector *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  const mxnet::TShape& a_shape = (*in_attrs)[0];
+  const int a_ndim = a_shape.ndim();
+
+  if (shape_is_known(a_shape)) {
+    CHECK_GT(a_shape.Size(), 0U)
+      << "Not support zero-size input array which has no identity";
+    if (a_ndim < 2) {
+      SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(0, 0));
+    } else {
+      mxnet::TShape rank_shape(a_ndim - 2, 0);
+      for (int i = 0; i < a_ndim - 2; ++i) { rank_shape[i] = a_shape[i]; }
+      SHAPE_ASSIGN_CHECK(*out_attrs, 0, rank_shape);
+    }
+  }
+  return shape_is_known(*in_attrs) && shape_is_known(*out_attrs);
+}
+
+inline bool MatrixRankNoneTolType(const nnvm::NodeAttrs& attrs,
+                                  std::vector<int>* in_attrs,
+                                  std::vector<int>* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  int a_type = in_attrs->at(0);
+
+  CHECK_NE(a_type, mshadow::kFloat16)
+    << "array type float16 is unsupported in linalg.";
+  CHECK(a_type == mshadow::kFloat32 || a_type == mshadow::kFloat64)
+    << "array type should be float32 or float64.";
+  TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kInt64);
+  return out_attrs->at(0) != -1;
+}
+
+DMLC_REGISTER_PARAMETER(MatrixRankNoneTolParam);
+
+NNVM_REGISTER_OP(_npi_matrix_rank_none_tol)
+.describe(R"code()code" ADD_FILELINE)
+.set_attr_parser(mxnet::op::ParamParser<MatrixRankNoneTolParam>)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames", [](const NodeAttrs& attrs){
+  return std::vector<std::string>{"M"};
+})
+.set_attr<mxnet::FInferShape>("FInferShape", MatrixRankNoneTolShape)
+.set_attr<nnvm::FInferType>("FInferType", MatrixRankNoneTolType)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs){
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.set_attr<FCompute>("FCompute<cpu>", MatrixRankNoneTolForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("M", "NDArray-or-Symbol", "Tensor of matrix")
+.add_arguments(MatrixRankNoneTolParam::__FIELDS__());
+
+inline bool MatrixRankShape(const nnvm::NodeAttrs& attrs,
+                            mxnet::ShapeVector *in_attrs,
+                            mxnet::ShapeVector *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  const mxnet::TShape& a_shape = (*in_attrs)[0];
+  const mxnet::TShape& tol_shape = (*in_attrs)[1];
+  const int a_ndim = a_shape.ndim();
+  const int tol_ndim = tol_shape.ndim();
+
+  if (shape_is_known(a_shape) && shape_is_known(tol_shape)) {
+    CHECK_GT(a_shape.Size(), 0U)
+      << "Not support zero-size input array which has no identity";
+    if (a_ndim < 2) {
+      SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(0, 0));
+    } else {
+      mxnet::TShape broadcast_shape;
+      GetOrCheckBroadcastShape(attrs, a_shape, tol_shape, &broadcast_shape);
+      if (broadcast_shape.ndim() == 1) {
+        if (tol_ndim == 0) {
+          SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(0, 0));
+        } else {
+          SHAPE_ASSIGN_CHECK(*out_attrs, 0, mxnet::TShape(1, 1));
+        }
+      } else {
+        mxnet::TShape rank_shape(broadcast_shape.ndim() - 1, 0);
+        for (int i = 0; i < broadcast_shape.ndim() - 1; ++i) {
+          rank_shape[i] = broadcast_shape[i];
+        }
+        SHAPE_ASSIGN_CHECK(*out_attrs, 0, rank_shape);
+      }
+    }
+  }
+  return shape_is_known(*in_attrs) && shape_is_known(*out_attrs);
+}
+
+inline bool MatrixRankType(const nnvm::NodeAttrs& attrs,
+                           std::vector<int>* in_attrs,
+                           std::vector<int>* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  int a_type = in_attrs->at(0);
+  int tol_type = in_attrs->at(1);
+
+  CHECK_NE(a_type, mshadow::kFloat16)
+    << "array type float16 is unsupported in linalg.";
+  CHECK(a_type == mshadow::kFloat32 || a_type == mshadow::kFloat64)
+    << "array type should be float32 or float64.";
+  CHECK(tol_type == mshadow::kFloat32 || tol_type == mshadow::kFloat64)
+    << "tol type should be float32 or float64.";
+  CHECK_EQ(a_type, tol_type)
+    << "array type and tol type should be the same.";
+  TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kInt64);
+  return out_attrs->at(0) != -1;
+}
+
+DMLC_REGISTER_PARAMETER(MatrixRankParam);
+
+NNVM_REGISTER_OP(_npi_matrix_rank)
+.describe(R"code()code" ADD_FILELINE)
+.set_attr_parser(mxnet::op::ParamParser<MatrixRankParam>)
+.set_num_inputs(2)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames", [](const NodeAttrs& attrs){
+  return std::vector<std::string>{"M", "tol"};
+})
+.set_attr<mxnet::FInferShape>("FInferShape", MatrixRankShape)
+.set_attr<nnvm::FInferType>("FInferType", MatrixRankType)
+.set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& attrs){
+  return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+})
+.set_attr<FCompute>("FCompute<cpu>", MatrixRankForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("M", "NDArray-or-Symbol", "Tensor of matrix")
+.add_argument("tol", "NDArray-or-Symbol", "Tensor of matrix")
+.add_arguments(MatrixRankParam::__FIELDS__());
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/linalg/np_matrix_rank.cu b/src/operator/numpy/linalg/np_matrix_rank.cu
new file mode 100644
index 000000000000..9528f698d35c
--- /dev/null
+++ b/src/operator/numpy/linalg/np_matrix_rank.cu
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_matrix_rank.cu
+ * \brief GPU implementation of the matrix_rank Operator
+ */
+#include <mxnet/operator_util.h>
+#include "./np_matrix_rank-inl.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_matrix_rank_none_tol)
+.set_attr<FCompute>("FCompute<gpu>", MatrixRankNoneTolForward<gpu>);
+
+NNVM_REGISTER_OP(_npi_matrix_rank)
+.set_attr<FCompute>("FCompute<gpu>", MatrixRankForward<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 3c0f01ee938e..ae69cccac799 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -2045,12 +2045,29 @@ def _add_workload_linalg_matrix_power():
 
 
 def _add_workload_linalg_matrix_rank():
-    a = np.eye(4)
-    b = a; b[-1,-1] = 0
-    c = np.ones((4,))
-    OpArgMngr.add_workload('linalg.matrix_rank', a)
-    OpArgMngr.add_workload('linalg.matrix_rank', b)
-    OpArgMngr.add_workload('linalg.matrix_rank', c)
+    shapes = [
+        ((4, 3), ()),
+        ((4, 3), (1,)),
+        ((4, 3), (2, 3,)),
+        ((2, 1, 1), (1,)),
+        ((2, 3, 3), (2,)),
+        ((2, 3, 1, 1), ()),
+        ((2, 3, 4, 4), (1, 3)),
+        ((2, 3, 4, 5), (2, 3)),
+        ((2, 3, 5, 4), (2, 3)),
+    ]
+    dtypes = (np.float32, np.float64)
+    for dtype in dtypes:
+        for a_shape, tol_shape in shapes:
+            for tol_is_none in [True, False]:
+                a_np = _np.asarray(_np.random.uniform(-10., 10., a_shape))
+                a = np.array(a_np, dtype=dtype)
+                if tol_is_none:
+                    OpArgMngr.add_workload('linalg.matrix_rank', a, None, False)
+                else:
+                    tol_np = _np.random.uniform(10., 20., tol_shape)
+                    tol = np.array(tol_np, dtype=dtype)
+                    OpArgMngr.add_workload('linalg.matrix_rank', a, tol, False)
 
 
 def _add_workload_linalg_multi_dot():
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 29ebd00a25ff..792fcb68a5a0 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -5741,6 +5741,83 @@ def check_lstsq(a_np, b_np, rcond_np, x, residuals, rank, s):
                 check_lstsq(a_np, b_np, rcond, x, residuals, rank, s)
 
 
+@with_seed()
+@use_np
+def test_np_linalg_matrix_rank():
+    class TestMatrixRank(HybridBlock):
+        def __init__(self, hermitian):
+            super(TestMatrixRank, self).__init__()
+            self._hermitian = hermitian
+
+        def hybrid_forward(self, F, M, tol=None):
+            return F.np.linalg.matrix_rank(M, tol, hermitian=self._hermitian)
+
+    def check_matrix_rank(rank, a_np, tol, hermitian):
+        try:
+            rank_expected = _np.linalg.matrix_rank(a_np, tol=tol, hermitian=hermitian)
+        except Exception as e:
+            print("a:", a_np)
+            print("a shape:", a_np.shape)
+            print(e)
+        else:
+            if a_np.ndim < 2:
+                assert rank.shape == _np.asarray(rank_expected).shape
+            else:
+                assert rank.shape == rank_expected.shape
+            assert_almost_equal(rank.asnumpy(), rank_expected, rtol=rtol, atol=atol)
+
+    shapes = [
+        ((), ()),
+        ((1,), (1,)),
+        ((3,), (1,)),
+        ((1, 1), ()),
+        ((1, 1), (1,)),
+        ((3, 3), (1,)),
+        ((3, 4), (1,)),
+        ((4, 3), ()),
+        ((4, 3), (1,)),
+        ((4, 3), (2,)),
+        ((4, 3), (2, 3,)),
+        ((2, 1, 1), ()),
+        ((2, 1, 1), (1,)),
+        ((2, 3, 3), (2,)),
+        ((2, 3, 4), (1,)),
+        ((2, 4, 3), (2,)),
+        ((2, 3, 1, 1), ()),
+        ((2, 3, 1, 1), (1, 1)),
+        ((2, 3, 1, 1), (2, 1)),
+        ((2, 3, 4, 4), (1, 3)),
+        ((2, 3, 4, 5), (2, 1)),
+        ((2, 3, 5, 4), (1, 3)),
+        ((2, 3, 1, 1), (2, 3)),
+        ((2, 3, 4, 4), (2, 3)),
+        ((2, 3, 4, 5), (2, 3)),
+        ((2, 3, 5, 4), (2, 3)),
+    ]
+    dtypes = ['float32', 'float64']
+    for dtype in dtypes:
+        for a_shape, tol_shape in shapes:
+            for tol_is_none, hybridize in itertools.product([True, False], [True, False]):
+                rtol = 1e-3
+                atol = 1e-5
+                test_matrix_rank = TestMatrixRank(hermitian=False)
+                if hybridize:
+                    test_matrix_rank.hybridize()
+
+                a_np = _np.asarray(_np.random.uniform(-10., 10., a_shape))
+                a = np.array(a_np, dtype=dtype)
+                if tol_is_none:
+                    rank = test_matrix_rank(a)
+                    # check matrix_rank validity
+                    check_matrix_rank(rank, a.asnumpy(), tol=None, hermitian=False)
+                else:
+                    tol_np = _np.random.uniform(10., 20., tol_shape)
+                    tol = np.array(tol_np, dtype=dtype)
+                    rank = test_matrix_rank(a, tol)
+                    # check matrix_rank validity
+                    check_matrix_rank(rank, a.asnumpy(), tol.asnumpy(), hermitian=False)
+
+
 @with_seed()
 @use_np
 def test_np_linalg_pinv():

From 20f70e4ed85b346146b1596fd31326c679af3510 Mon Sep 17 00:00:00 2001
From: hanke580 <38852697+hanke580@users.noreply.github.com>
Date: Tue, 24 Mar 2020 03:41:58 +0800
Subject: [PATCH 34/44] [Numpy] Kron operator (#17323)

* [Numpy]Add kron

* Implement the forward of Kron op

* Implement the Backward of a

* Implement the Backward of b

* Fix 3rd party

* Fix cpp sanity

* Finish grad check

* address comments: fix test_np_op and reduce req to req[0]

* * Fix  ndim  = 0

* * Fix uninitialize bugs

* * Impl FFI
---
 benchmark/python/ffi/benchmark_ffi.py         |   1 +
 python/mxnet/ndarray/numpy/_op.py             |  47 ++-
 python/mxnet/numpy/multiarray.py              |  48 ++-
 python/mxnet/numpy_dispatch_protocol.py       |   1 +
 python/mxnet/symbol/numpy/_symbol.py          |  48 ++-
 src/api/operator/numpy/np_kron.cc             |  44 +++
 src/operator/numpy/np_kron-inl.h              | 322 ++++++++++++++++++
 src/operator/numpy/np_kron.cc                 |  94 +++++
 src/operator/numpy/np_kron.cu                 |  37 ++
 .../unittest/test_numpy_interoperability.py   |   8 +
 tests/python/unittest/test_numpy_op.py        |  81 +++++
 11 files changed, 728 insertions(+), 3 deletions(-)
 create mode 100644 src/api/operator/numpy/np_kron.cc
 create mode 100644 src/operator/numpy/np_kron-inl.h
 create mode 100644 src/operator/numpy/np_kron.cc
 create mode 100644 src/operator/numpy/np_kron.cu

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 2711cf9d06e3..45067a73c638 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -58,6 +58,7 @@ def prepare_workloads():
     OpArgMngr.add_workload("ediff1d", pool['2x2'], pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("nan_to_num", pool['2x2'])
     OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
+    OpArgMngr.add_workload("kron", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.eig", pool['3x3'])
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 7192f0e5824c..2b1af0b1daaf 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -42,7 +42,7 @@
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
-           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
+           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
@@ -6081,6 +6081,51 @@ def outer(a, b):
     return tensordot(a.flatten(), b.flatten(), 0)
 
 
+@set_module('mxnet.ndarray.numpy')
+def kron(a, b):
+    r"""
+    Kronecker product of two arrays.
+    Computes the Kronecker product, a composite array made of blocks of the
+    second array scaled by the first.
+
+    Parameters
+    ----------
+    a, b : ndarray
+
+    Returns
+    -------
+    out : ndarray
+
+    See Also
+    --------
+    outer : The outer product
+
+    Notes
+    -----
+    The function assumes that the number of dimensions of `a` and `b`
+    are the same, if necessary prepending the smallest with ones.
+    If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,
+    the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.
+    The elements are products of elements from `a` and `b`, organized
+    explicitly by::
+        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
+    where::
+        kt = it * st + jt,  t = 0,...,N
+    In the common 2-D case (N=1), the block structure can be visualized::
+        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],
+        [  ...                              ...   ],
+        [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]
+
+    Examples
+    --------
+    >>> np.kron([1,10,100], [5,6,7])
+    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])
+    >>> np.kron([5,6,7], [1,10,100])
+    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])
+    """
+    return _api_internal.kron(a, b)
+
+
 @set_module('mxnet.ndarray.numpy')
 def vdot(a, b):
     r"""
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 0f539f1ec8c6..901a37228b53 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -65,7 +65,8 @@
            'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman',
            'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',
            'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',
-           'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'equal', 'not_equal',
+           'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
+           'equal', 'not_equal',
            'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
@@ -7920,6 +7921,51 @@ def outer(a, b):
     return tensordot(a.flatten(), b.flatten(), 0)
 
 
+@set_module('mxnet.numpy')
+def kron(a, b):
+    r"""
+    Kronecker product of two arrays.
+    Computes the Kronecker product, a composite array made of blocks of the
+    second array scaled by the first.
+
+    Parameters
+    ----------
+    a, b : ndarray
+
+    Returns
+    -------
+    out : ndarray
+
+    See Also
+    --------
+    outer : The outer product
+
+    Notes
+    -----
+    The function assumes that the number of dimensions of `a` and `b`
+    are the same, if necessary prepending the smallest with ones.
+    If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,
+    the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.
+    The elements are products of elements from `a` and `b`, organized
+    explicitly by::
+        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
+    where::
+        kt = it * st + jt,  t = 0,...,N
+    In the common 2-D case (N=1), the block structure can be visualized::
+        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],
+        [  ...                              ...   ],
+        [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]
+
+    Examples
+    --------
+    >>> np.kron([1,10,100], [5,6,7])
+    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])
+    >>> np.kron([5,6,7], [1,10,100])
+    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])
+    """
+    return _mx_nd_np.kron(a, b)
+
+
 @set_module('mxnet.numpy')
 def vdot(a, b):
     r"""
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index 9c955e5702b5..c32d99b05feb 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -171,6 +171,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'tril',
     'meshgrid',
     'outer',
+    'kron',
     'einsum',
     'polyval',
     'shares_memory',
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index fff778a90f1d..a2f613a09437 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -47,7 +47,7 @@
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
-           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer',
+           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
@@ -5575,6 +5575,52 @@ def outer(a, b):
     return tensordot(a.flatten(), b.flatten(), 0)
 
 
+@set_module('mxnet.symbol.numpy')
+def kron(a, b):
+    r"""
+    kron(a, b)
+    Kronecker product of two arrays.
+    Computes the Kronecker product, a composite array made of blocks of the
+    second array scaled by the first.
+
+    Parameters
+    ----------
+    a, b : ndarray
+
+    Returns
+    -------
+    out : ndarray
+
+    See Also
+    --------
+    outer : The outer product
+
+    Notes
+    -----
+    The function assumes that the number of dimensions of `a` and `b`
+    are the same, if necessary prepending the smallest with ones.
+    If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`,
+    the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`.
+    The elements are products of elements from `a` and `b`, organized
+    explicitly by::
+        kron(a,b)[k0,k1,...,kN] = a[i0,i1,...,iN] * b[j0,j1,...,jN]
+    where::
+        kt = it * st + jt,  t = 0,...,N
+    In the common 2-D case (N=1), the block structure can be visualized::
+        [[ a[0,0]*b,   a[0,1]*b,  ... , a[0,-1]*b  ],
+        [  ...                              ...   ],
+        [ a[-1,0]*b,  a[-1,1]*b, ... , a[-1,-1]*b ]]
+
+    Examples
+    --------
+    >>> np.kron([1,10,100], [5,6,7])
+    array([  5,   6,   7,  50,  60,  70, 500, 600, 700])
+    >>> np.kron([5,6,7], [1,10,100])
+    array([  5,  50, 500,   6,  60, 600,   7,  70, 700])
+    """
+    return _npi.kron(a, b)
+
+
 @set_module('mxnet.symbol.numpy')
 def vdot(a, b):
     r"""
diff --git a/src/api/operator/numpy/np_kron.cc b/src/api/operator/numpy/np_kron.cc
new file mode 100644
index 000000000000..753798208b4f
--- /dev/null
+++ b/src/api/operator/numpy/np_kron.cc
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_kron.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_kron.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_kron-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.kron")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = Op::Get("_npi_kron");
+  attrs.op = op;
+  NDArray* inputs[] = {args[0].operator NDArray*(), args[1].operator NDArray*()};
+  int num_inputs = 2;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_kron-inl.h b/src/operator/numpy/np_kron-inl.h
new file mode 100644
index 000000000000..0d72921691a9
--- /dev/null
+++ b/src/operator/numpy/np_kron-inl.h
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_kron-inl.h
+ * \brief Function definition of matrix numpy-compatible kron operator
+ */
+#ifndef MXNET_OPERATOR_NUMPY_NP_KRON_INL_H_
+#define MXNET_OPERATOR_NUMPY_NP_KRON_INL_H_
+
+#include <vector>
+#include "np_tensordot_op-inl.h"
+#include "../mxnet_op.h"
+
+namespace mxnet {
+namespace op {
+
+template<int ndim, int req>
+struct kron {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* out,
+                                  const DType* a, const DType* b,
+                                  mshadow::Shape<ndim> ashape,
+                                  mshadow::Shape<ndim> bshape,
+                                  mshadow::Shape<ndim> oshape) {
+    using namespace mxnet_op;
+
+    auto k = unravel(i, oshape);
+    Shape<ndim> ia;
+    Shape<ndim> jb;
+    for (int q = 0; q < ndim; q++) {
+      ia[q] = static_cast<int>(k[q] / bshape[q]);
+      jb[q] = k[q] % bshape[q];
+    }
+    auto idx_a = ravel(ia, ashape);
+    auto idx_b = ravel(jb, bshape);
+
+    KERNEL_ASSIGN(out[i], req, a[idx_a] * b[idx_b]);
+  }
+};
+
+template<int ndim, int req>
+struct kron_back_a {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* agrad,
+                                  const DType* b, const DType* ograd,
+                                  mshadow::Shape<ndim> ashape,
+                                  mshadow::Shape<ndim> bshape,
+                                  mshadow::Shape<ndim> oshape) {
+    using namespace mxnet_op;
+
+    auto ia = unravel(i, ashape);
+    Shape<ndim> k;
+    DType temp_agrad = 0;
+
+    for (int idx_b = 0; idx_b < bshape.Size(); idx_b++) {
+      auto jb = unravel(idx_b, bshape);
+      for (int q = 0; q < ndim; q++) {
+        k[q] = ia[q]*bshape[q] + jb[q];
+      }
+      auto idx_o = ravel(k, oshape);
+      temp_agrad += b[idx_b]*ograd[idx_o];
+    }
+    KERNEL_ASSIGN(agrad[i], req, temp_agrad);
+  }
+};
+
+template<int ndim, int req>
+struct kron_back_b {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, const DType* a,
+                                  DType* bgrad, const DType* ograd,
+                                  mshadow::Shape<ndim> ashape,
+                                  mshadow::Shape<ndim> bshape,
+                                  mshadow::Shape<ndim> oshape) {
+    using namespace mxnet_op;
+
+    auto jb = unravel(i, bshape);
+    Shape<ndim> k;
+    DType temp_bgrad = 0;
+
+    for (int idx_a = 0; idx_a < ashape.Size(); idx_a++) {
+      auto ia = unravel(idx_a, ashape);
+      for (int q = 0; q < ndim; q++) {
+        k[q] = ia[q] * bshape[q] + jb[q];
+      }
+      auto idx_o = ravel(k, oshape);
+      temp_bgrad += a[idx_a]*ograd[idx_o];
+    }
+    KERNEL_ASSIGN(bgrad[i], req, temp_bgrad);
+  }
+};
+
+template<typename xpu>
+void KronOpForwardImpl(const OpContext& ctx,
+                       OpReqType req,
+                       const TBlob& a,
+                       const TBlob& b,
+                       const TBlob& out
+                       ) {
+  using namespace mshadow;
+
+  if (req == kNullOp) {
+    return;
+  }
+
+  if (out.shape_.Size() == 0U) {
+    return;  // zero-size output, no need to launch kernel
+  }
+
+  const mxnet::TShape& ashape = a.shape_;
+  const mxnet::TShape& bshape = b.shape_;
+  const mxnet::TShape& oshape = out.shape_;
+
+
+  // TensordotIntAxesImpl<xpu>(0, ctx, a, b, out, req[0]);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  MSHADOW_TYPE_SWITCH(out.type_flag_, DType, {
+    if (ashape.Size() == 0U || bshape.Size() == 0U) {
+      // 0-size input
+      if (req != kAddTo) {
+        Tensor<xpu, 1, DType> out_data = out.get_with_shape<xpu, 1, DType>(
+            Shape1(out.shape_.Size()), s);
+        out_data = static_cast<DType>(0);
+      }
+    } else if (ashape.ndim() == 0 && bshape.ndim() == 0) {
+      // Both 0-D scalars, equivalent to multiply
+      Tensor<xpu, 1, DType> a_data = a.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> b_data = b.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> out_data = out.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      ASSIGN_DISPATCH(out_data, req, a_data * b_data);
+    } else if (ashape.ndim() == 0 || bshape.ndim() == 0) {
+      // Either of them is a scalar, just scale by one of them
+      const DType* tensor = (ashape.ndim() == 0) ? b.dptr<DType>() : a.dptr<DType>();
+      const DType* scalar = (ashape.ndim() == 0) ? a.dptr<DType>() : b.dptr<DType>();
+      MXNET_ASSIGN_REQ_SWITCH(req, Req, {
+        mxnet_op::Kernel<scalar_mul_kernel<Req>, xpu>::Launch(
+          s, out.Size(), out.dptr<DType>(), tensor, scalar);
+      });
+    } else {
+      MXNET_NDIM_SWITCH(oshape.ndim(), ndim, {
+        Shape<ndim> ashape_ = oshape.get<ndim>();
+        Shape<ndim> bshape_ = oshape.get<ndim>();
+        Shape<ndim> oshape_ = oshape.get<ndim>();
+        int temp = ashape.ndim()-bshape.ndim();
+        int s_dim = (temp > 0)?bshape.ndim():ashape.ndim();
+        for (int i = 0; i < s_dim; i++) {
+          ashape_[ndim - i - 1] = ashape[ashape.ndim() - i - 1];
+          bshape_[ndim - i - 1] = bshape[bshape.ndim() - i - 1];
+          oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+        }
+        if (temp > 0) {
+          for (int i = s_dim; i < ndim; i++) {
+            ashape_[ndim - i - 1] = ashape[ashape.ndim() - i - 1];
+            bshape_[ndim - i - 1] = 1;
+            oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+          }
+        } else {
+          for (int i = s_dim; i < ndim; i++) {
+            ashape_[ndim - i - 1] = 1;
+            bshape_[ndim - i - 1] = bshape[bshape.ndim() - i - 1];
+            oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+          }
+        }
+        MXNET_ASSIGN_REQ_SWITCH(req, req_type, {
+          mxnet_op::Kernel<kron<ndim, req_type>, xpu>::Launch(
+            s, out.Size(), out.dptr<DType>(), a.dptr<DType>(), b.dptr<DType>(),
+            ashape_, bshape_, oshape_);
+        });
+      });
+    }
+  });
+}
+
+template<typename xpu>
+void KronOpBackwardImpl(const OpContext& ctx,
+                        const std::vector<OpReqType>& req,
+                        const TBlob& a,
+                        const TBlob& b,
+                        const TBlob& ograd,
+                        const TBlob& agrad,
+                        const TBlob& bgrad) {
+  const mxnet::TShape& ashape = a.shape_;
+  const mxnet::TShape& bshape = b.shape_;
+  const mxnet::TShape& oshape = ograd.shape_;
+
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  MSHADOW_TYPE_SWITCH(ograd.type_flag_, DType, {
+    if (ashape.ndim() == 0 && bshape.ndim() == 0) {
+      // Both 0-D scalars, equivalent to multiply
+      Tensor<xpu, 1, DType> ograd_data = ograd.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> a_data = a.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> b_data = b.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> agrad_data = agrad.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> bgrad_data = bgrad.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      ASSIGN_DISPATCH(agrad_data, req[0], b_data * ograd_data);
+      ASSIGN_DISPATCH(bgrad_data, req[1], a_data * ograd_data);
+    } else if (ashape.ndim() == 0 || bshape.ndim() == 0) {
+      // Either of them is a scalar, just scale by one of them
+      const TBlob& tensor = (ashape.ndim() == 0) ? b : a;
+      const TBlob& tensor_grad = (ashape.ndim() == 0) ? bgrad : agrad;
+      const TBlob& scalar = (ashape.ndim() == 0) ? a : b;
+      const TBlob& scalar_grad = (ashape.ndim() == 0) ? agrad : bgrad;
+      Tensor<xpu, 1, DType> scalar_ = scalar.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> scalar_grad_ = scalar_grad.get_with_shape<xpu, 1, DType>(Shape1(1), s);
+      Tensor<xpu, 1, DType> tensor_ = tensor.FlatTo1D<xpu, DType>(s);
+      Tensor<xpu, 1, DType> tensor_grad_ = tensor_grad.FlatTo1D<xpu, DType>(s);
+      Tensor<xpu, 1, DType> ograd_ = ograd.FlatTo1D<xpu, DType>(s);
+      const OpReqType& tensor_req = (ashape.ndim() == 0) ? req[1] : req[0];
+      const OpReqType& scalar_req = (ashape.ndim() == 0) ? req[0] : req[1];
+      ASSIGN_DISPATCH(tensor_grad_, tensor_req,
+                      broadcast_scalar(scalar_, tensor_grad_.shape_) * ograd_);
+      Tensor<xpu, 1, DType> workspace =
+        ctx.requested[0].get_space_typed<xpu, 1, DType>(Shape1(ograd.shape_.Size()), s);
+      ASSIGN_DISPATCH(workspace, kWriteTo, tensor_ * ograd_);
+
+      ReduceAxesComputeImpl<xpu, mshadow_op::sum, true>(
+        ctx, {TBlob(workspace)}, {scalar_req}, {TBlob(scalar_grad_)}, scalar_grad_.shape_);
+    } else {
+      MXNET_NDIM_SWITCH(oshape.ndim(), ndim, {
+        Shape<ndim> ashape_ = oshape.get<ndim>();
+        Shape<ndim> bshape_ = oshape.get<ndim>();
+        Shape<ndim> oshape_ = oshape.get<ndim>();
+        int temp = ashape.ndim()-bshape.ndim();
+        int s_dim =  (temp > 0)?bshape.ndim():ashape.ndim();
+        for (int i = 0; i < s_dim; i++) {
+          ashape_[ndim - i - 1] = ashape[ashape.ndim() - i - 1];
+          bshape_[ndim - i - 1] = bshape[bshape.ndim() - i - 1];
+          oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+        }
+        if (temp > 0) {
+          for (int i = s_dim; i < ndim; i++) {
+            ashape_[ndim - i - 1] = ashape[ashape.ndim() - i - 1];
+            bshape_[ndim - i - 1] = 1;
+            oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+          }
+        } else {
+          for (int i = s_dim; i < ndim; i++) {
+            ashape_[ndim - i - 1] = 1;
+            bshape_[ndim - i - 1] = bshape[bshape.ndim() - i - 1];
+            oshape_[ndim - i - 1] = oshape[oshape.ndim() - i - 1];
+          }
+        }
+        MSHADOW_TYPE_SWITCH(agrad.type_flag_, DType, {
+          MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+            mxnet_op::Kernel<kron_back_a<ndim, req_type>, xpu>::Launch(
+              s, agrad.Size(), agrad.dptr<DType>(), b.dptr<DType>(), ograd.dptr<DType>(),
+              ashape_, bshape_, oshape_);
+          });
+        });
+        MSHADOW_TYPE_SWITCH(bgrad.type_flag_, DType, {
+          MXNET_ASSIGN_REQ_SWITCH(req[1], req_type, {
+            mxnet_op::Kernel<kron_back_b<ndim, req_type>, xpu>::Launch(
+              s, bgrad.Size(), a.dptr<DType>(), bgrad.dptr<DType>(), ograd.dptr<DType>(),
+              ashape_, bshape_, oshape_);
+          });
+        });
+      });
+    }
+  });
+}
+
+template<typename xpu>
+inline void KronOpForward(const nnvm::NodeAttrs& attrs,
+                          const OpContext& ctx,
+                          const std::vector<TBlob>& inputs,
+                          const std::vector<OpReqType>& req,
+                          const std::vector<TBlob>& outputs) {
+  using namespace mshadow;
+
+  CHECK_EQ(inputs.size(), 2U);
+  CHECK_EQ(outputs.size(), 1U);
+
+  const TBlob& a = inputs[0];
+  const TBlob& b = inputs[1];
+  const TBlob& out = outputs[0];
+
+  KronOpForwardImpl<xpu>(ctx, req[0], a, b, out);
+}
+
+
+template<typename xpu>
+inline void KronOpBackward(const nnvm::NodeAttrs& attrs,
+                           const OpContext& ctx,
+                           const std::vector<TBlob>& inputs,
+                           const std::vector<OpReqType>& req,
+                           const std::vector<TBlob>& outputs) {
+  using namespace mxnet_op;
+  using namespace mshadow;
+
+  CHECK_EQ(inputs.size(), 3U);
+  CHECK_EQ(outputs.size(), 2U);
+
+  const TBlob& ograd = inputs[0];
+  const TBlob& a = inputs[1];
+  const TBlob& b = inputs[2];
+  const TBlob& grad_a = outputs[0];
+  const TBlob& grad_b = outputs[1];
+
+  KronOpBackwardImpl<xpu>(ctx, req, a, b, ograd, grad_a, grad_b);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_NP_KRON_INL_H_
diff --git a/src/operator/numpy/np_kron.cc b/src/operator/numpy/np_kron.cc
new file mode 100644
index 000000000000..321e51bc259e
--- /dev/null
+++ b/src/operator/numpy/np_kron.cc
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_kron.cc
+ * \brief CPU Implementation of numpy-compatible Kronecker product
+ */
+
+#include "./np_kron-inl.h"
+
+namespace mxnet {
+namespace op {
+
+inline bool KronOpShape(const nnvm::NodeAttrs& attrs,
+                        mxnet::ShapeVector *in_attrs,
+                        mxnet::ShapeVector *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  const mxnet::TShape& a_shape = in_attrs->at(0);
+  const mxnet::TShape& b_shape = in_attrs->at(1);
+
+  if (!ndim_is_known(a_shape) || !ndim_is_known(b_shape)) {
+    return false;
+  }
+
+  mxnet::TShape out_shape(std::max(a_shape.ndim(), b_shape.ndim()), -1);
+  if (a_shape.ndim() > b_shape.ndim()) {
+    for (int i = 0; i < a_shape.ndim() - b_shape.ndim(); i++) {
+      out_shape[i] = a_shape[i];
+    }
+    for (int i = a_shape.ndim() - b_shape.ndim(); i < a_shape.ndim(); i++) {
+      out_shape[i] = a_shape[i] * b_shape[i - a_shape.ndim() + b_shape.ndim()];
+    }
+  } else {
+    for (int i = 0; i < b_shape.ndim() - a_shape.ndim(); i++) {
+      out_shape[i] = b_shape[i];
+    }
+    for (int i = b_shape.ndim() - a_shape.ndim(); i < b_shape.ndim(); i++) {
+      out_shape[i] = b_shape[i] * a_shape[i - b_shape.ndim() + a_shape.ndim()];
+    }
+  }
+
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, out_shape);
+
+  return shape_is_known(*in_attrs) && shape_is_known(*out_attrs);
+}
+
+NNVM_REGISTER_OP(_npi_kron)
+.set_num_inputs(2)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::string>{"a", "b"};
+  })
+.set_attr<mxnet::FInferShape>("FInferShape", KronOpShape)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<2, 1>)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", KronOpForward<cpu>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_kron"})
+.add_argument("a", "NDArray-or-Symbol", "First input")
+.add_argument("b", "NDArray-or-Symbol", "Second input");
+
+NNVM_REGISTER_OP(_backward_npi_kron)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", KronOpBackward<cpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_kron.cu b/src/operator/numpy/np_kron.cu
new file mode 100644
index 000000000000..fc2fb1f765b9
--- /dev/null
+++ b/src/operator/numpy/np_kron.cu
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_kron.cu
+ * \brief GPU Implementation of numpy-compatible Kronecker product
+ */
+
+#include "./np_kron-inl.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_kron)
+.set_attr<FCompute>("FCompute<gpu>", KronOpForward<gpu>);
+
+NNVM_REGISTER_OP(_backward_npi_kron)
+.set_attr<FCompute>("FCompute<gpu>", KronOpBackward<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index ae69cccac799..75a86afac8d2 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -1266,6 +1266,13 @@ def _add_workload_outer():
     OpArgMngr.add_workload('outer', np.ones((5)), np.ones((2)))
 
 
+def _add_workload_kron():
+    OpArgMngr.add_workload('kron', np.ones((5)), np.ones((2)))
+    OpArgMngr.add_workload('kron', np.arange(16).reshape((4,4)), np.ones((4,4)))
+    OpArgMngr.add_workload('kron', np.ones((2,4)), np.zeros((2,4)))
+    OpArgMngr.add_workload('kron', np.ones(()), np.ones(()))
+
+
 def _add_workload_meshgrid():
     OpArgMngr.add_workload('meshgrid', np.array([1, 2, 3]))
     OpArgMngr.add_workload('meshgrid', np.array([1, 2, 3]), np.array([4, 5, 6, 7]))
@@ -2850,6 +2857,7 @@ def _prepare_workloads():
     _add_workload_trace()
     _add_workload_tril()
     _add_workload_outer()
+    _add_workload_kron()
     _add_workload_meshgrid()
     _add_workload_einsum()
     _add_workload_abs()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 792fcb68a5a0..59e18a832b78 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -555,6 +555,87 @@ def ShapeReduce(mat, shape, is_b=False):
         assert_raises(MXNetError, lambda: np.matmul(a, b))
 
 
+@with_seed()
+@use_np
+def test_np_kron():
+    def np_kron_backward(ograd, a, b):
+        ndim = ograd.ndim
+        # Make ndim equal
+        if ndim > a.ndim:
+            a = a.reshape((1,)*(ndim - a.ndim) + a.shape)
+        else:
+            b = b.reshape((1,)*(ndim - b.ndim) + b.shape)
+        assert(a.ndim == b.ndim)
+
+        # Compute agrad
+        agrad = _np.zeros(a.shape)
+        for i in range(a.size):
+            ia = _np.asarray(_np.unravel_index(i, a.shape))
+            for j in range(b.size):
+                jb = _np.asarray(_np.unravel_index(j, b.shape))
+                k = ia * _np.asarray(b.shape) + jb
+                agrad[tuple(ia)] += ograd[tuple(k)] * b[tuple(jb)]
+        # Compute bgrad
+        bgrad = _np.zeros(b.shape)
+        for j in range(b.size):
+            jb = _np.asarray(_np.unravel_index(j, b.shape))
+            for i in range(a.size):
+                ia = _np.asarray(_np.unravel_index(i, a.shape))
+                k = ia * _np.asarray(b.shape) + jb
+                bgrad[tuple(jb)] += ograd[tuple(k)] * a[tuple(ia)]
+        return [agrad, bgrad]
+
+    class TestKron(HybridBlock):
+        def __init__(self):
+            super(TestKron, self).__init__()
+
+        def hybrid_forward(self, F, a, b):
+            return F.np.kron(a, b)
+
+    # test input
+    tensor_shapes = [
+        ((3,), (3,)),
+        ((2, 3), (3,)),
+        ((2, 3, 4), (2,)),
+        ((3, 2), ())
+    ]
+
+    for hybridize in [True, False]:
+        for a_shape, b_shape in tensor_shapes:
+            for dtype in [_np.float32, _np.float64]:
+                test_kron = TestKron()
+                if hybridize:
+                    test_kron.hybridize()
+                a = rand_ndarray(shape=a_shape, dtype=dtype).as_np_ndarray()
+                b = rand_ndarray(shape=b_shape, dtype=dtype).as_np_ndarray()
+                a.attach_grad()
+                b.attach_grad()
+
+                np_out = _np.kron(a.asnumpy(), b.asnumpy())
+                with mx.autograd.record():
+                    mx_out = test_kron(a, b)
+                assert mx_out.shape == np_out.shape
+                assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False)
+                mx_out.backward()
+
+                # Test imperative once again
+                mx_out = np.kron(a, b)
+                np_out = _np.kron(a.asnumpy(), b.asnumpy())
+                assert_almost_equal(mx_out.asnumpy(), np_out, rtol=1e-3, atol=1e-5, use_broadcast=False)
+
+                # test numeric gradient
+                a_sym = mx.sym.Variable("a").as_np_ndarray()
+                b_sym = mx.sym.Variable("b").as_np_ndarray()
+                mx_sym = mx.sym.np.kron(a_sym, b_sym).as_nd_ndarray()
+                check_numeric_gradient(mx_sym, [a.as_nd_ndarray(), b.as_nd_ndarray()],
+                                       rtol=1e-2, atol=1e-2, dtype=dtype)
+
+                # test gradient via backward implemented by numpy
+                np_backward = np_kron_backward(_np.ones(np_out.shape, dtype = dtype), a.asnumpy(), b.asnumpy())
+                assert_almost_equal(a.grad.asnumpy(), np_backward[0], rtol=1e-2, atol=1e-2)
+                assert_almost_equal(b.grad.asnumpy(), np_backward[1], rtol=1e-2, atol=1e-2)
+
+
 @with_seed()
 @use_np
 def test_np_sum():

From 2f692521e370eea5b9429e7246514e28522dbd14 Mon Sep 17 00:00:00 2001
From: Minghao Liu <40382964+Tommliu@users.noreply.github.com>
Date: Thu, 9 Apr 2020 07:58:42 +0800
Subject: [PATCH 35/44] [Numpy] OP_interp (#17793)

* interp

* fix_uninitialized_issue
---
 python/mxnet/ndarray/numpy/_op.py             |  82 ++++-
 python/mxnet/numpy/fallback.py                |   2 -
 python/mxnet/numpy/multiarray.py              |  79 ++++-
 python/mxnet/numpy_dispatch_protocol.py       |   1 +
 python/mxnet/symbol/numpy/_symbol.py          |  55 +++-
 src/api/operator/numpy/np_interp_op.cc        |  78 +++++
 src/operator/numpy/np_interp_op-inl.h         | 286 ++++++++++++++++++
 src/operator/numpy/np_interp_op.cc            |  92 ++++++
 src/operator/numpy/np_interp_op.cu            |  34 +++
 .../unittest/test_numpy_interoperability.py   |  33 ++
 tests/python/unittest/test_numpy_op.py        |  67 ++++
 11 files changed, 804 insertions(+), 5 deletions(-)
 create mode 100644 src/api/operator/numpy/np_interp_op.cc
 create mode 100644 src/operator/numpy/np_interp_op-inl.h
 create mode 100644 src/operator/numpy/np_interp_op.cc
 create mode 100644 src/operator/numpy/np_interp_op.cu

diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 2b1af0b1daaf..b0a934f10680 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -44,7 +44,7 @@
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
            'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
-           'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory',
+           'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'interp',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
            'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'diag', 'diagonal']
@@ -7018,6 +7018,86 @@ def may_share_memory(a, b, max_work=None):
     return _api_internal.share_memory(a, b).item()
 
 
+@set_module('mxnet.ndarray.numpy')
+def interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments
+    """
+    One-dimensional linear interpolation.
+    Returns the one-dimensional piecewise linear interpolant to a function
+    with given values at discrete data-points.
+
+    Parameters
+    ----------
+    x : ndarray
+        The x-coordinates of the interpolated values.
+    xp : 1-D array of floats
+        The x-coordinates of the data points, must be increasing if argument
+        `period` is not specified. Otherwise, `xp` is internally sorted after
+        normalizing the periodic boundaries with ``xp = xp % period``.
+    fp : 1-D array of floats
+        The y-coordinates of the data points, same length as `xp`.
+    left : optional float corresponding to fp
+        Value to return for `x < xp[0]`, default is `fp[0]`.
+    right : optional float corresponding to fp
+        Value to return for `x > xp[-1]`, default is `fp[-1]`.
+    period : None or float, optional
+        A period for the x-coordinates. This parameter allows the proper
+        interpolation of angular x-coordinates. Parameters `left` and `right`
+        are ignored if `period` is specified.
+        .. versionadded:: 1.10.0
+
+    Returns
+    -------
+    y : float (corresponding to fp) or ndarray
+        The interpolated values, same shape as `x`.
+    Raises
+    ------
+    ValueError
+        If `xp` and `fp` have different length
+        If `xp` or `fp` are not 1-D sequences
+        If `period == 0`
+
+    Notes
+    -----
+    Does not check that the x-coordinate sequence `xp` is increasing.
+    If `xp` is not increasing, the results are nonsense.
+    A simple check for increasing is::
+        np.all(np.diff(xp) > 0)
+
+    Examples
+    --------
+    >>> xp = [1, 2, 3]
+    >>> fp = [3, 2, 0]
+    >>> np.interp(2.5, xp, fp)
+    1.0
+    >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp)
+    array([ 3. ,  3. ,  2.5 ,  0.56,  0. ])
+    >>> UNDEF = -99.0
+    >>> np.interp(3.14, xp, fp, right=UNDEF)
+    -99.0
+    Plot an interpolant to the sine function:
+    >>> x = np.linspace(0, 2*np.pi, 10)
+    >>> y = np.sin(x)
+    >>> xvals = np.linspace(0, 2*np.pi, 50)
+    >>> yinterp = np.interp(xvals, x, y)
+    >>> import matplotlib.pyplot as plt
+    >>> plt.plot(x, y, 'o')
+    [<matplotlib.lines.Line2D object at 0x...>]
+    >>> plt.plot(xvals, yinterp, '-x')
+    [<matplotlib.lines.Line2D object at 0x...>]
+    >>> plt.show()
+    Interpolation with periodic x-coordinates:
+    >>> x = [-180, -170, -185, 185, -10, -5, 0, 365]
+    >>> xp = [190, -190, 350, -350]
+    >>> fp = [5, 10, 3, 4]
+    >>> np.interp(x, xp, fp, period=360)
+    array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75])
+    """
+    if not isinstance(x, numeric_types):
+        x = x.astype(float)
+    return _api_internal.interp(xp.astype(float), fp.astype(float), x, left,
+                                right, period)
+
+
 @set_module('mxnet.ndarray.numpy')
 def diff(a, n=1, axis=-1, prepend=None, append=None):  # pylint: disable=redefined-outer-name
     r"""
diff --git a/python/mxnet/numpy/fallback.py b/python/mxnet/numpy/fallback.py
index 7dddb100f2fa..295014a569a0 100644
--- a/python/mxnet/numpy/fallback.py
+++ b/python/mxnet/numpy/fallback.py
@@ -50,7 +50,6 @@
     'histogramdd',
     'i0',
     'in1d',
-    'interp',
     'intersect1d',
     'isclose',
     'isin',
@@ -137,7 +136,6 @@
 histogramdd = onp.histogramdd
 i0 = onp.i0
 in1d = onp.in1d
-interp = onp.interp
 intersect1d = onp.intersect1d
 isclose = onp.isclose
 isin = onp.isin
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 901a37228b53..f2883cc1df6b 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -66,7 +66,7 @@
            'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',
            'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',
            'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
-           'equal', 'not_equal',
+           'equal', 'not_equal', 'interp',
            'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
@@ -9160,6 +9160,83 @@ def resize(a, new_shape):
     return _mx_nd_np.resize(a, new_shape)
 
 
+@set_module('mxnet.numpy')
+def interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments
+    """
+    One-dimensional linear interpolation.
+    Returns the one-dimensional piecewise linear interpolant to a function
+    with given values at discrete data-points.
+
+    Parameters
+    ----------
+    x : ndarray
+        The x-coordinates of the interpolated values.
+    xp : 1-D array of floats
+        The x-coordinates of the data points, must be increasing if argument
+        `period` is not specified. Otherwise, `xp` is internally sorted after
+        normalizing the periodic boundaries with ``xp = xp % period``.
+    fp : 1-D array of floats
+        The y-coordinates of the data points, same length as `xp`.
+    left : optional float corresponding to fp
+        Value to return for `x < xp[0]`, default is `fp[0]`.
+    right : optional float corresponding to fp
+        Value to return for `x > xp[-1]`, default is `fp[-1]`.
+    period : None or float, optional
+        A period for the x-coordinates. This parameter allows the proper
+        interpolation of angular x-coordinates. Parameters `left` and `right`
+        are ignored if `period` is specified.
+        .. versionadded:: 1.10.0
+
+    Returns
+    -------
+    y : float (corresponding to fp) or ndarray
+        The interpolated values, same shape as `x`.
+    Raises
+    ------
+    ValueError
+        If `xp` and `fp` have different length
+        If `xp` or `fp` are not 1-D sequences
+        If `period == 0`
+
+    Notes
+    -----
+    Does not check that the x-coordinate sequence `xp` is increasing.
+    If `xp` is not increasing, the results are nonsense.
+    A simple check for increasing is::
+        np.all(np.diff(xp) > 0)
+
+    Examples
+    --------
+    >>> xp = [1, 2, 3]
+    >>> fp = [3, 2, 0]
+    >>> np.interp(2.5, xp, fp)
+    1.0
+    >>> np.interp([0, 1, 1.5, 2.72, 3.14], xp, fp)
+    array([ 3. ,  3. ,  2.5 ,  0.56,  0. ])
+    >>> UNDEF = -99.0
+    >>> np.interp(3.14, xp, fp, right=UNDEF)
+    -99.0
+    Plot an interpolant to the sine function:
+    >>> x = np.linspace(0, 2*np.pi, 10)
+    >>> y = np.sin(x)
+    >>> xvals = np.linspace(0, 2*np.pi, 50)
+    >>> yinterp = np.interp(xvals, x, y)
+    >>> import matplotlib.pyplot as plt
+    >>> plt.plot(x, y, 'o')
+    [<matplotlib.lines.Line2D object at 0x...>]
+    >>> plt.plot(xvals, yinterp, '-x')
+    [<matplotlib.lines.Line2D object at 0x...>]
+    >>> plt.show()
+    Interpolation with periodic x-coordinates:
+    >>> x = [-180, -170, -185, 185, -10, -5, 0, 365]
+    >>> xp = [190, -190, 350, -350]
+    >>> fp = [5, 10, 3, 4]
+    >>> np.interp(x, xp, fp, period=360)
+    array([7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75])
+    """
+    return _mx_nd_np.interp(x, xp, fp, left=left, right=right, period=period)
+
+
 # pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def full_like(a, fill_value, dtype=None, order='C', ctx=None, out=None): # pylint: disable=too-many-arguments
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index c32d99b05feb..a2dd3b313c8f 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -110,6 +110,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'fliplr',
     'inner',
     'insert',
+    'interp',
     'max',
     'amax',
     'mean',
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index a2f613a09437..8d3893efe22c 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -46,7 +46,7 @@
            'average', 'mean', 'maximum', 'minimum', 'any', 'all', 'around', 'round', 'round_', 'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
-           'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
+           'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'interp',
            'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
@@ -6298,6 +6298,59 @@ def ediff1d(ary, to_end=None, to_begin=None):
                             to_begin_scalar=to_begin, to_end_scalar=to_end)
 
 
+@set_module('mxnet.symbol.numpy')
+def interp(x, xp, fp, left=None, right=None, period=None):  # pylint: disable=too-many-arguments
+    """
+    One-dimensional linear interpolation.
+    Returns the one-dimensional piecewise linear interpolant to a function
+    with given values at discrete data-points.
+
+    Parameters
+    ----------
+    x : _Symbol
+        The x-coordinates of the interpolated values.
+    xp : _Symbol
+        The x-coordinates of the data points, must be increasing if argument
+        `period` is not specified. Otherwise, `xp` is internally sorted after
+        normalizing the periodic boundaries with ``xp = xp % period``.
+    fp : _Symbol
+        The y-coordinates of the data points, same length as `xp`.
+    left : optional float corresponding to fp
+        Value to return for `x < xp[0]`, default is `fp[0]`.
+    right : optional float corresponding to fp
+        Value to return for `x > xp[-1]`, default is `fp[-1]`.
+    period : None or float, optional
+        A period for the x-coordinates. This parameter allows the proper
+        interpolation of angular x-coordinates. Parameters `left` and `right`
+        are ignored if `period` is specified.
+        .. versionadded:: 1.10.0
+
+    Returns
+    -------
+    y : _Symbol
+        The interpolated values, same shape as `x`.
+
+    Raises
+    ------
+    ValueError
+        If `xp` and `fp` have different length
+        If `xp` or `fp` are not 1-D sequences
+        If `period == 0`
+
+    Notes
+    -----
+    Does not check that the x-coordinate sequence `xp` is increasing.
+    If `xp` is not increasing, the results are nonsense.
+    A simple check for increasing is::
+        np.all(np.diff(xp) > 0)
+    """
+    if isinstance(x, numeric_types):
+        return _npi.interp(xp.astype(float), fp.astype(float), left=left,
+                           right=right, period=period, x_scalar=x, x_is_scalar=True)
+    return _npi.interp(xp.astype(float), fp.astype(float), x.astype(float), left=left,
+                       right=right, period=period, x_scalar=0.0, x_is_scalar=False)
+
+
 @set_module('mxnet.symbol.numpy')
 def resize(a, new_shape):
     """
diff --git a/src/api/operator/numpy/np_interp_op.cc b/src/api/operator/numpy/np_interp_op.cc
new file mode 100644
index 000000000000..7959383c9230
--- /dev/null
+++ b/src/api/operator/numpy/np_interp_op.cc
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_interp_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_interp_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_interp_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.interp")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  static const nnvm::Op* op = Op::Get("_npi_interp");
+  nnvm::NodeAttrs attrs;
+  op::NumpyInterpParam param;
+  if (args[3].type_code() == kNull) {
+    param.left = dmlc::nullopt;
+  } else {
+    param.left = args[3].operator double();
+  }
+  if (args[4].type_code() == kNull) {
+    param.right = dmlc::nullopt;
+  } else {
+    param.right = args[4].operator double();
+  }
+  if (args[5].type_code() == kNull) {
+    param.period = dmlc::nullopt;
+  } else {
+    param.period = args[5].operator double();
+  }
+  if (args[2].type_code() == kDLInt || args[2].type_code() == kDLFloat) {
+    param.x_scalar = args[2].operator double();
+    param.x_is_scalar = true;
+    attrs.op = op;
+    attrs.parsed = std::move(param);
+    SetAttrDict<op::NumpyInterpParam>(&attrs);
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*()};
+    int num_inputs = 2;
+    int num_outputs = 0;
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  } else {
+    param.x_scalar = 0.0;
+    param.x_is_scalar = false;
+    attrs.op = op;
+    attrs.parsed = std::move(param);
+    SetAttrDict<op::NumpyInterpParam>(&attrs);
+    NDArray* inputs[] = {args[0].operator mxnet::NDArray*(), args[1].operator mxnet::NDArray*(),
+                         args[2].operator mxnet::NDArray*()};
+    int num_inputs = 3;
+    int num_outputs = 0;
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, nullptr);
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_interp_op-inl.h b/src/operator/numpy/np_interp_op-inl.h
new file mode 100644
index 000000000000..c0d595699a82
--- /dev/null
+++ b/src/operator/numpy/np_interp_op-inl.h
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_interp_op-inl.h
+*/
+
+#ifndef MXNET_OPERATOR_NUMPY_NP_INTERP_OP_INL_H_
+#define MXNET_OPERATOR_NUMPY_NP_INTERP_OP_INL_H_
+
+#include <vector>
+#include <string>
+#include <unordered_map>
+#include "../tensor/ordering_op-inl.h"
+#include "../tensor/matrix_op-inl.h"
+#include "../tensor/elemwise_binary_scalar_op.h"
+#include "../../common/utils.h"
+#include "../mshadow_op.h"
+#include "../operator_common.h"
+#include "../elemwise_op_common.h"
+#include "np_broadcast_reduce_op.h"
+
+namespace mxnet {
+namespace op {
+
+struct NumpyInterpParam : public dmlc::Parameter<NumpyInterpParam> {
+  dmlc::optional<double> left;
+  dmlc::optional<double> right;
+  dmlc::optional<double> period;
+  double x_scalar;
+  bool x_is_scalar;
+  DMLC_DECLARE_PARAMETER(NumpyInterpParam) {
+    DMLC_DECLARE_FIELD(left)
+      .set_default(dmlc::optional<double>())
+      .describe("Value to return for x < xp[0], default is fp[0].");
+    DMLC_DECLARE_FIELD(right)
+      .set_default(dmlc::optional<double>())
+      .describe("Value to return for x > xp[-1], default is fp[-1].");
+    DMLC_DECLARE_FIELD(period)
+      .set_default(dmlc::optional<double>())
+      .describe("A period for the x-coordinates. This parameter allows"
+                "the proper interpolation of angular x-coordinates. Parameters"
+                "left and right are ignored if period is specified.");
+    DMLC_DECLARE_FIELD(x_scalar).set_default(0.0)
+      .describe("x is a scalar input");
+    DMLC_DECLARE_FIELD(x_is_scalar).set_default(false)
+      .describe("Flag that determines whether input is a scalar");
+  }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream left_s, right_s, period_s, x_scalar_s, x_is_scalar_s;
+    left_s << left;
+    right_s << right;
+    period_s << period;
+    x_scalar_s << x_scalar;
+    x_is_scalar_s << x_is_scalar;
+    (*dict)["left"] = left_s.str();
+    (*dict)["right"] = right_s.str();
+    (*dict)["period"] = period_s.str();
+    (*dict)["x_scalar"] = x_scalar_s.str();
+    (*dict)["x_is_scalar"] = x_is_scalar_s.str();
+  }
+};
+
+struct interp {
+  MSHADOW_XINLINE static void Map(int i,
+                                  double* out,
+                                  const double* x,
+                                  const double* xp,
+                                  const double* fp,
+                                  const int dsize,
+                                  const double left,
+                                  const double right,
+                                  const bool has_left,
+                                  const bool has_right) {
+    double x_value = x[i];
+    double xp_low = xp[0];
+    double xp_above = xp[dsize-1];
+    double lval = has_left ? left : fp[0];
+    double rval = has_right ? right : fp[dsize-1];
+
+    if (x_value > xp_above) {
+      out[i] = rval;
+    } else if (x_value < xp_low) {
+      out[i] = lval;
+    } else {
+      int imin = 0;
+      int imax = dsize;
+      int imid;
+      while (imin < imax) {
+        imid = static_cast<int>((imax + imin) / 2);
+        if (x_value >= xp[imid]) {
+          imin = imid + 1;
+        } else {
+          imax = imid;
+        }
+      }  // biserction search
+
+      int j = imin;
+      if (j == dsize) {
+        out[i] = fp[dsize-1];
+      } else if (x_value == xp[j-1]) {
+        out[i] = fp[j-1];  // void potential non-finite interpolation
+      } else {
+        double xp_below = xp[j-1];
+        double xp_above = xp[j];
+        double weight_above = (x_value - xp_below) / (xp_above - xp_below);
+        double weigth_below = 1 - weight_above;
+        double x1 = fp[j-1] * weigth_below;
+        double x2 = fp[j] * weight_above;
+        out[i] = x1 + x2;
+      }
+    }
+  }
+};
+
+struct interp_period {
+  MSHADOW_XINLINE static void Map(int i,
+                                  double* out,
+                                  const double* x,
+                                  const double* xp,
+                                  const double* fp,
+                                  const index_t* idx,
+                                  const int dsize,
+                                  const double period) {
+    double x_value = x[i];
+    int imin = 0;
+    int imax = dsize;
+    int imid;
+    while (imin < imax) {
+      imid = static_cast<int>((imax + imin) / 2);
+      if (x_value >= xp[idx[imid]]) {
+        imin = imid + 1;
+      } else {
+        imax = imid;
+      }
+    }  // biserction search
+
+    int j = imin;
+    double xp_below, xp_above;
+    double fp1, fp2;
+    if (j == 0) {
+      xp_below = xp[idx[dsize-1]] - period;
+      xp_above = xp[idx[0]];
+      fp1 = fp[idx[dsize-1]];
+      fp2 = fp[idx[0]];
+    } else if (j == dsize) {
+      xp_below = xp[idx[dsize-1]];
+      xp_above = xp[idx[0]] + period;
+      fp1 = fp[idx[dsize-1]];
+      fp2 = fp[idx[0]];
+    } else {
+      xp_below = xp[idx[j-1]];
+      xp_above = xp[idx[j]];
+      fp1 = fp[idx[j-1]];
+      fp2 = fp[idx[j]];
+    }
+    double weight_above = (x_value - xp_below) / (xp_above - xp_below);
+    double weigth_below = 1 - weight_above;
+    double x1 = fp1 * weigth_below;
+    double x2 = fp2 * weight_above;
+    out[i] = x1 + x2;
+  }
+};
+
+template<typename xpu, typename OP>
+void NumpyInterpForward(const nnvm::NodeAttrs& attrs,
+                        const OpContext &ctx,
+                        const std::vector<TBlob> &inputs,
+                        const std::vector<OpReqType> &req,
+                        const std::vector<TBlob> &outputs) {
+  if (req[0] == kNullOp) return;
+  using namespace mxnet;
+  using namespace mxnet_op;
+  using namespace mshadow;
+  using namespace mshadow::expr;
+  CHECK_GE(inputs.size(), 2U);
+  CHECK_EQ(outputs.size(), 1U);
+
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const NumpyInterpParam& param = nnvm::get<NumpyInterpParam>(attrs.parsed);
+  dmlc::optional<double> left = param.left;
+  dmlc::optional<double> right = param.right;
+  dmlc::optional<double> period = param.period;
+  bool x_is_scalar = param.x_is_scalar;
+
+  TBlob xp = inputs[0];
+  const TBlob &fp = inputs[1];
+  const TBlob &out = outputs[0];
+  bool has_left = left.has_value() ? true : false;
+  bool has_right = right.has_value() ? true : false;
+  double left_value = left.has_value() ? left.value() : 0.0;
+  double right_value = right.has_value() ? right.value() : 0.0;
+
+  CHECK_GE(xp.Size(), 1U) <<"ValueError: array of sample points is empty";
+
+  TopKParam topk_param = TopKParam();
+  topk_param.axis = dmlc::optional<int>(-1);
+  topk_param.is_ascend = true;
+  topk_param.k = 0;
+  topk_param.ret_typ = topk_enum::kReturnIndices;
+
+  size_t topk_temp_size;  // Used by Sort
+  size_t topk_workspace_size = TopKWorkspaceSize<xpu, double>(xp, topk_param, &topk_temp_size);
+  size_t size_x = x_is_scalar ? 8 : 0;
+  size_t size_norm_x = x_is_scalar ? 8 : inputs[2].Size() * sizeof(double);
+  size_t size_norm_xp = xp.Size() * sizeof(double);
+  size_t size_norm = period.has_value()? size_norm_x + size_norm_xp : 0;
+  size_t size_idx = period.has_value()? xp.Size() * sizeof(index_t) : 0;
+  size_t workspace_size =
+    topk_workspace_size + size_x + size_norm + size_idx;
+
+  Tensor<xpu, 1, char> temp_mem =
+    ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(workspace_size), s);
+
+  char* workspace_curr_ptr = temp_mem.dptr_;
+
+  TBlob x, idx;
+  if (x_is_scalar) {
+    double x_scalar = param.x_scalar;
+    Tensor<cpu, 1, double> host_x(&x_scalar, Shape1(1), ctx.get_stream<cpu>());
+    Tensor<xpu, 1, double> device_x(reinterpret_cast<double*>(workspace_curr_ptr),
+                                    Shape1(1), ctx.get_stream<xpu>());
+    Copy(device_x, host_x, ctx.get_stream<xpu>());
+    x = TBlob(device_x.dptr_, TShape(0, 1), xpu::kDevMask);
+    workspace_curr_ptr += 8;
+  } else {
+    x = inputs[2];
+  }   // handle input x is a scalar
+
+  // normalize the input data by periodic boundaries.
+  if (period.has_value()) {
+    double* norm_xp_ptr;
+    double* norm_x_ptr;
+    double period_value = period.value();
+    index_t* idx_ptr;
+    CHECK_NE(period_value, 0.0)<< "period must be a non-zero value";
+
+    norm_xp_ptr = reinterpret_cast<double*>(workspace_curr_ptr);
+    norm_x_ptr = reinterpret_cast<double*>(workspace_curr_ptr + size_norm_xp);
+    idx_ptr = reinterpret_cast<index_t*>(workspace_curr_ptr + size_norm_xp + size_norm_x);
+
+    TBlob norm_x = TBlob(norm_x_ptr, x.shape_, xpu::kDevMask);
+    TBlob norm_xp = TBlob(norm_xp_ptr, xp.shape_, xpu::kDevMask);
+    const OpReqType ReqType = kWriteTo;
+    Kernel<op_with_req<OP, ReqType>, xpu>::Launch(
+      s, x.Size(), norm_x.dptr<double>(), x.dptr<double>(), period_value);
+    Kernel<op_with_req<OP, ReqType>, xpu>::Launch(
+      s, xp.Size(), norm_xp.dptr<double>(), xp.dptr<double>(), period_value);
+
+    workspace_curr_ptr += size_x + size_norm + size_idx;
+    idx = TBlob(idx_ptr, xp.shape_, xpu::kDevMask);
+    std::vector<OpReqType> req_TopK = {kWriteTo};
+    std::vector<TBlob> ret = {idx};
+
+    TopKImplwithWorkspace<xpu, double, index_t>(ctx.run_ctx, req_TopK, norm_xp, ret, topk_param,
+                                                workspace_curr_ptr, topk_temp_size, s);
+    Kernel<interp_period, xpu>::Launch(
+      s, norm_x.Size(), out.dptr<double>(), norm_x.dptr<double>(), norm_xp.dptr<double>(),
+      fp.dptr<double>(), idx.dptr<index_t>(), norm_xp.Size(), period_value);
+  } else {
+    Kernel<interp, xpu>::Launch(
+      s, x.Size(), out.dptr<double>(), x.dptr<double>(), xp.dptr<double>(), fp.dptr<double>(),
+      xp.Size(), left_value, right_value, has_left, has_right);
+  }
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_NP_INTERP_OP_INL_H_
diff --git a/src/operator/numpy/np_interp_op.cc b/src/operator/numpy/np_interp_op.cc
new file mode 100644
index 000000000000..af4bb1cd0bb3
--- /dev/null
+++ b/src/operator/numpy/np_interp_op.cc
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_interp_op.cc
+ * \brief CPU Implementation of Numpy-compatible interp
+*/
+
+#include "np_interp_op-inl.h"
+
+namespace mxnet {
+namespace op {
+
+inline bool NumpyInterpShape(const nnvm::NodeAttrs& attrs,
+                             std::vector<TShape> *in_attrs,
+                             std::vector<TShape> *out_attrs) {
+  CHECK_GE(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+  const NumpyInterpParam& param = nnvm::get<NumpyInterpParam>(attrs.parsed);
+
+  TShape oshape;
+  CHECK_EQ(in_attrs->at(0).ndim(), 1U)
+    << "ValueError: Data points must be 1-D array";
+  CHECK_EQ(in_attrs->at(1).ndim(), 1U)
+    << "ValueError: Data points must be 1-D array";
+  CHECK_EQ(in_attrs->at(0)[0], in_attrs->at(1)[0])
+    << "ValueError: fp and xp are not of the same length";
+  oshape = param.x_is_scalar ? TShape(0, 1) : in_attrs->at(2);
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape);
+  return shape_is_known(out_attrs->at(0));
+}
+
+inline bool NumpyInterpType(const nnvm::NodeAttrs& attrs,
+                            std::vector<int> *in_attrs,
+                            std::vector<int> *out_attrs) {
+  CHECK_GE(in_attrs->size(), 2U);
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kFloat64);
+  return out_attrs->at(0) != -1;
+}
+
+DMLC_REGISTER_PARAMETER(NumpyInterpParam);
+
+NNVM_REGISTER_OP(_npi_interp)
+.set_num_inputs([](const NodeAttrs& attrs) {
+  const NumpyInterpParam& param =
+    nnvm::get<NumpyInterpParam>(attrs.parsed);
+  return param.x_is_scalar ? 2 : 3;
+  })
+.set_num_outputs(1)
+.set_attr_parser(ParamParser<NumpyInterpParam>)
+.set_attr<mxnet::FInferShape>("FInferShape", NumpyInterpShape)
+.set_attr<nnvm::FInferType>("FInferType", NumpyInterpType)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    const NumpyInterpParam& param =
+      nnvm::get<NumpyInterpParam>(attrs.parsed);
+    return param.x_is_scalar ?
+           std::vector<std::string>{"xp", "fp"} :
+           std::vector<std::string>{"xp", "fp", "x"};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyInterpForward<cpu, mshadow_op::mod>)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<THasDeterministicOutput>("THasDeterministicOutput", true)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("xp", "NDArray-or-Symbol", "Input x-coordinates")
+.add_argument("fp", "NDArray-or-Symbol", "Input y-coordinates")
+.add_argument("x", "NDArray-or-Symbol", "Input data")
+.add_arguments(NumpyInterpParam::__FIELDS__());
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_interp_op.cu b/src/operator/numpy/np_interp_op.cu
new file mode 100644
index 000000000000..d3753e3eab0f
--- /dev/null
+++ b/src/operator/numpy/np_interp_op.cu
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_interp_op.cu
+ * \brief GPU Implementation of Numpy-compatible interp
+*/
+
+#include "np_interp_op-inl.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_interp)
+.set_attr<FCompute>("FCompute<gpu>", NumpyInterpForward<gpu, mshadow_op::mod>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 75a86afac8d2..4e58786df643 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -1362,6 +1362,38 @@ def _add_workload_insert():
     OpArgMngr.add_workload('insert', np.array(1), 0, np.array([0]))
 
 
+def _add_workload_interp():
+    xp0 = np.linspace(0, 1, 5)
+    fp0 = np.linspace(0, 1, 5)
+    x0 = np.linspace(0, 1, 50)
+    xp1 = np.array([1, 2, 3, 4])
+    fp1 = np.array([1, 2, np.inf, 4])
+    x1 = np.array([1, 2, 2.5, 3, 4])
+    xp2 = np.arange(0, 10, 0.0001)
+    fp2 = np.sin(xp2)
+    xp3 = np.array([190, -190, 350, -350])
+    fp3 = np.array([5, 10, 3, 4])
+    x3 = np.array([-180, -170, -185, 185, -10, -5, 0, 365])
+
+    OpArgMngr.add_workload('interp', x0, xp0, fp0)
+    OpArgMngr.add_workload('interp', x1, xp1, fp1)
+    OpArgMngr.add_workload('interp', np.pi, xp2, fp2)
+    OpArgMngr.add_workload('interp', x3, xp3, fp3, period=360)
+    for size in range(1, 10):
+        xp = np.arange(size, dtype=np.float64)
+        fp = np.ones(size, dtype=np.float64)
+        incpts = np.array([-1, 0, size - 1, size], dtype=np.float64)
+        decpts = incpts[::-1]
+        OpArgMngr.add_workload('interp', incpts, xp, fp)
+        OpArgMngr.add_workload('interp', decpts, xp, fp)
+        OpArgMngr.add_workload('interp', incpts, xp, fp, left=0)
+        OpArgMngr.add_workload('interp', decpts, xp, fp, left=0)
+        OpArgMngr.add_workload('interp', incpts, xp, fp, right=2)
+        OpArgMngr.add_workload('interp', decpts, xp, fp, right=2)
+        OpArgMngr.add_workload('interp', incpts, xp, fp, left=0, right=2)
+        OpArgMngr.add_workload('interp', decpts, xp, fp, left=0, right=2)
+
+
 def _add_workload_hypot():
     OpArgMngr.add_workload('hypot', np.array(1), np.array(1))
     OpArgMngr.add_workload('hypot', np.array(0), np.array(0))
@@ -2869,6 +2901,7 @@ def _prepare_workloads():
     _add_workload_true_divide()
     _add_workload_inner()
     _add_workload_insert()
+    _add_workload_interp()
     _add_workload_hypot()
     _add_workload_lcm()
     _add_workload_bitwise_and()
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 59e18a832b78..c51440b82426 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -8823,6 +8823,73 @@ def hybrid_forward(self, F, a):
             assert_almost_equal(elem_mx.asnumpy(), elem_np, rtol=rtol, atol=atol)
 
 
+@with_seed()
+@use_np
+def test_np_interp():
+    class TestInterp(HybridBlock):
+        def __init__(self, left=None, right=None, period=None):
+            super(TestInterp, self).__init__()
+            self._left = left
+            self._right = right
+            self._period = period
+        
+        def hybrid_forward(self, F, x, xp, fp):
+            return F.np.interp(x, xp, fp, left=self._left, right=self._right, period=self._period)
+    
+    class TestInterpScalar(HybridBlock):
+        def __init__(self, x=None, left=None, right=None, period=None):
+            super(TestInterpScalar, self).__init__()
+            self._x = x
+            self._left = left
+            self._right = right
+            self._period = period
+        
+        def hybrid_forward(self, F, xp, fp):
+            return F.np.interp(self._x, xp, fp, left=self._left, right=self._right, period=self._period)
+
+    xtypes = [np.int64, np.float32, np.float64]
+    dtypes = [np.int32, np.int64, np.float32, np.float64]
+    xshapes = [
+        (), (3,), (5,), (20,),
+        (2, 2), (4, 4), (8, 8),
+        (5, 5, 5), (8, 0, 8)
+    ]
+    dsizes = [10, 30]
+    periods = [None, 2*np.pi]
+    lefts = [None, -10, 0]
+    rights= [None, 20, 50]
+    flags = [True, False]
+    combinations = itertools.product(flags, flags, xshapes, dsizes, xtypes, dtypes, lefts, rights, periods)
+    for hybridize, x_scalar, xshape, dsize, xtype, dtype, left, right, period in combinations:
+        rtol = 1e-3
+        atol = 1e-5
+        if period is not None:
+            x = np.random.uniform(-np.pi, np.pi, size=xshape).astype(xtype)
+            xp = np.random.uniform(0, 2*np.pi, size=dsize)
+            fp = np.sin(xp)
+        else:
+            x = np.random.uniform(0, 100, size=xshape).astype(xtype)
+            xp = np.sort(np.random.choice(100, dsize, replace=False).astype(dtype))
+            fp = np.random.uniform(-50, 50, size=dsize).astype(dtype) 
+        np_x = x.asnumpy()
+        if x_scalar and xshape == ():
+            x = x.item()
+            np_x = x
+            test_interp = TestInterpScalar(x=x, left=left, right=right, period=period)
+        else: 
+            test_interp = TestInterp(left=left, right=right, period=period)
+        if hybridize:
+            test_interp.hybridize()
+        mx_out = test_interp(xp, fp) if (x_scalar and xshape == ()) else test_interp(x, xp, fp)
+        np_out = _np.interp(np_x, xp.asnumpy(), fp.asnumpy(), left=left, right=right, period=period)
+        assert mx_out.shape == np_out.shape
+        assert_almost_equal(mx_out.asnumpy(), np_out, atol=atol, rtol=rtol)
+
+        mx_out = np.interp(x, xp, fp, left=left, right=right, period=period)
+        np_out = _np.interp(np_x ,xp.asnumpy(), fp.asnumpy(), left=left, right=right, period=period)
+        assert_almost_equal(mx_out.asnumpy(), np_out, atol=atol, rtol=rtol)
+
+
 @with_seed()
 @use_np
 def test_np_bincount():

From 6fe903b7bd402f7e5ae7f0e3e4730c4f1f5e4dbc Mon Sep 17 00:00:00 2001
From: Yiyan66 <57363390+Yiyan66@users.noreply.github.com>
Date: Thu, 2 Apr 2020 12:03:36 +0800
Subject: [PATCH 36/44]  [numpy] add op median #17084

---
 3rdparty/mkldnn                               |  2 +-
 python/mxnet/ndarray/numpy/_op.py             | 51 ++++++++++++++++++-
 python/mxnet/numpy/multiarray.py              | 51 ++++++++++++++++++-
 python/mxnet/numpy_dispatch_protocol.py       |  1 +
 python/mxnet/symbol/numpy/_symbol.py          | 39 +++++++++++++-
 .../unittest/test_numpy_interoperability.py   |  9 ++++
 tests/python/unittest/test_numpy_op.py        | 40 +++++++++++++++
 7 files changed, 189 insertions(+), 4 deletions(-)

diff --git a/3rdparty/mkldnn b/3rdparty/mkldnn
index 07579e6c0c68..1b05a28eb966 160000
--- a/3rdparty/mkldnn
+++ b/3rdparty/mkldnn
@@ -1 +1 @@
-Subproject commit 07579e6c0c6839a390a6f3040e05a2b2c71e628a
+Subproject commit 1b05a28eb9666efef83b281e4cc1936db5e6cf6c
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index b0a934f10680..b04391125647 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -35,7 +35,7 @@
            'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul',
            'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram',
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'all', 'any', 'sort',
-           'tensordot', 'eye', 'linspace',
+           'tensordot', 'eye', 'linspace', 'median',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
            'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero',
@@ -6863,6 +6863,55 @@ def percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='l
     return _api_internal.percentile(a, q, axis, interpolation, keepdims, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+def median(a, axis=None, out=None, overwrite_input=None, keepdims=False):
+    r"""
+    Compute the median along the specified axis.
+    Returns the median of the array elements.
+    Parameters
+    ----------
+    a : array_like
+        Input array or object that can be converted to an array.
+    axis : {int, sequence of int, None}, optional
+        Axis or axes along which the medians are computed. The default
+        is to compute the median along a flattened version of the array.
+        A sequence of axes is supported since version 1.9.0.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output,
+        but the type (of the output) will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
+    Returns
+    -------
+    median : ndarray
+        A new array holding the result. If the input contains integers
+        or floats smaller than ``float32``, then the output data-type is
+        ``np.float32``.  Otherwise, the data-type of the output is the
+        same as that of the input. If `out` is specified, that array is
+        returned instead.
+    See Also
+    --------
+    mean, percentile
+    Examples
+    --------
+    >>> a = np.array([[10, 7, 4], [3, 2, 1]])
+    >>> a
+    array([[10,  7,  4],
+        [ 3,  2,  1]])
+    >>> np.median(a)
+    3.5
+    >>> np.median(a, axis=0)
+    array([6.5, 4.5, 2.5])
+    >>> np.median(a, axis=1)
+    array([7.,  2.])
+    """
+    return quantile(a=a, q=0.5, axis=axis, out=out, overwrite_input=overwrite_input,
+                    interpolation='midpoint', keepdims=keepdims)
+
+
 @set_module('mxnet.ndarray.numpy')
 def quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments
     """
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index f2883cc1df6b..38a9241138a1 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -51,7 +51,7 @@
 from . import fallback
 
 
-__all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape',
+__all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape', 'median',
            'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'all', 'any', 'broadcast_to',
            'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'bitwise_not', 'delete',
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'invert',
@@ -8866,6 +8866,55 @@ def percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='l
                                 interpolation=interpolation, keepdims=keepdims)
 
 
+@set_module('mxnet.numpy')
+def median(a, axis=None, out=None, overwrite_input=None, keepdims=False):
+    r"""
+    Compute the median along the specified axis.
+    Returns the median of the array elements.
+    Parameters
+    ----------
+    a : array_like
+        Input array or object that can be converted to an array.
+    axis : {int, sequence of int, None}, optional
+        Axis or axes along which the medians are computed. The default
+        is to compute the median along a flattened version of the array.
+        A sequence of axes is supported since version 1.9.0.
+    out : ndarray, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output,
+        but the type (of the output) will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
+    Returns
+    -------
+    median : ndarray
+        A new array holding the result. If the input contains integers
+        or floats smaller than ``float32``, then the output data-type is
+        ``np.float32``.  Otherwise, the data-type of the output is the
+        same as that of the input. If `out` is specified, that array is
+        returned instead.
+    See Also
+    --------
+    mean, percentile
+    Examples
+    --------
+    >>> a = np.array([[10, 7, 4], [3, 2, 1]])
+    >>> a
+    array([[10,  7,  4],
+        [ 3,  2,  1]])
+    >>> np.median(a)
+    3.5
+    >>> np.median(a, axis=0)
+    array([6.5, 4.5, 2.5])
+    >>> np.median(a, axis=1)
+    array([7.,  2.])
+    """
+    return _mx_nd_np.median(a, axis=axis, overwrite_input=overwrite_input,
+                            keepdims=keepdims, out=out)
+
+
 @set_module('mxnet.numpy')
 def quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments
     """
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index a2dd3b313c8f..c768dbc7a194 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -178,6 +178,7 @@ def _run_with_array_ufunc_proto(*args, **kwargs):
     'shares_memory',
     'may_share_memory',
     'quantile',
+    'median',
     'percentile',
     'diff',
     'ediff1d',
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 8d3893efe22c..df82c2598c60 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -38,7 +38,7 @@
 __all__ = ['zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'empty_like', 'bitwise_not', 'invert',
            'delete', 'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'arctan2',
            'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp',
-           'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'log1p', 'matmul',
+           'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'log1p', 'matmul', 'median',
            'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram', 'insert',
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
@@ -6112,6 +6112,43 @@ def percentile(a, q, axis=None, out=None, overwrite_input=None, interpolation='l
                            keepdims=keepdims, q_scalar=None, out=out)
 
 
+@set_module('mxnet.symbol.numpy')
+def median(a, axis=None, out=None, overwrite_input=None, keepdims=False):
+    r"""
+    Compute the median along the specified axis.
+    Returns the median of the array elements.
+    Parameters
+    ----------
+    a : _Symbol
+        Input array or object that can be converted to an array.
+    axis : {int, sequence of int, None}, optional
+        Axis or axes along which the medians are computed. The default
+        is to compute the median along a flattened version of the array.
+        A sequence of axes is supported since version 1.9.0.
+    out :  _Symbol, optional
+        Alternative output array in which to place the result. It must
+        have the same shape and buffer length as the expected output,
+        but the type (of the output) will be cast if necessary.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the original `arr`.
+    Returns
+    -------
+    median :  _Symbol
+        A new array holding the result. If the input contains integers
+        or floats smaller than ``float32``, then the output data-type is
+        ``np.float32``.  Otherwise, the data-type of the output is the
+        same as that of the input. If `out` is specified, that array is
+        returned instead.
+    See Also
+    --------
+    mean, percentile
+    """
+    return quantile(a=a, q=0.5, axis=axis, out=out, overwrite_input=overwrite_input,
+                    interpolation='midpoint', keepdims=keepdims)
+
+
 @set_module('mxnet.symbol.numpy')
 def quantile(a, q, axis=None, out=None, overwrite_input=None, interpolation='linear', keepdims=False): # pylint: disable=too-many-arguments
     """
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 4e58786df643..e88bf88a10ab 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -178,6 +178,14 @@ def _add_workload_diagonal():
     OpArgMngr.add_workload('diagonal', B, 0, 2, 1)
 
 
+def _add_workload_median(array_pool):
+    OpArgMngr.add_workload('median', array_pool['4x1'])
+    OpArgMngr.add_workload('median', array_pool['4x1'], axis=0, keepdims=True)
+    OpArgMngr.add_workload('median', np.array([[1, 2, 3], [4, 5, 6]]))
+    OpArgMngr.add_workload('median', np.array([[1, 2, 3], [4, 5, 6]]), axis=0)
+    OpArgMngr.add_workload('median', np.array([[1, 2, 3], [4, 5, 6]]), axis=1)
+
+
 def _add_workload_quantile():
     x1 = np.arange(8) * 0.5
     x2 = np.arange(100.)
@@ -2962,6 +2970,7 @@ def _prepare_workloads():
     _add_workload_diff()
     _add_workload_ediff1d()
     _add_workload_quantile()
+    _add_workload_median(array_pool)
     _add_workload_percentile()
     _add_workload_resize()
     _add_workload_full_like(array_pool)
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index c51440b82426..9e4b777c77d7 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -7841,6 +7841,46 @@ def test_np_share_memory():
                 assert not op(np.ones((5, 0), dtype=dt), np.ones((0, 3, 0), dtype=adt))
 
 
+def test_np_median():
+    class TestMedian(HybridBlock):
+        def __init__(self, axis=None, keepdims=False):
+            super(TestMedian, self).__init__()
+            self._axis = axis
+            self._keepdims = keepdims
+
+        def hybrid_forward(self, F, a):
+            return F.np.median(a, axis=self._axis, keepdims=self._keepdims)
+
+    flags = [True, False]
+    dtypes = ['float16', 'float32', 'float64']
+    qtypes = ['float32', 'float64']
+    tensor_shapes = [
+        ((2, 3), None),
+        ((2, 3, 4, 5), 3),
+        ((2, 3, 4), (0, 2)),
+        ((2, 3, 4), 1)
+    ]
+
+    for hybridize, keepdims, (a_shape, axis), dtype in \
+        itertools.product(flags, flags, tensor_shapes, dtypes):
+        atol = 3e-4 if dtype == 'float16' else 1e-4
+        rtol = 3e-2 if dtype == 'float16' else 1e-2
+        test_median = TestMedian(axis=axis, keepdims=keepdims)
+        if hybridize:
+            test_median.hybridize()
+        a = np.random.uniform(-1.0, 1.0, size=a_shape)
+        np_out = _np.median(a.asnumpy(), axis=axis, keepdims=keepdims)
+        mx_out = test_median(a)
+        
+        assert mx_out.shape == np_out.shape
+        assert_almost_equal(mx_out.asnumpy(), np_out, atol=atol, rtol=rtol)
+
+        mx_out = np.median(a, axis=axis, keepdims=keepdims)
+        np_out = _np.median(a.asnumpy(), axis=axis, keepdims=keepdims)
+
+        assert_almost_equal(mx_out.asnumpy(), np_out, atol=atol, rtol=rtol)
+
+
 @with_seed()
 @use_np
 def test_np_quantile():

From 3f195fbab25eb2c8723f3bd96eb621d72840f2da Mon Sep 17 00:00:00 2001
From: hanke580 <38852697+hanke580@users.noreply.github.com>
Date: Mon, 23 Mar 2020 15:36:39 +0800
Subject: [PATCH 37/44] [Numpy] Add op fmax, fmin, fmod (#17567)

---
 benchmark/python/ffi/benchmark_ffi.py         |   3 +
 python/mxnet/ndarray/numpy/_op.py             |  76 +++++++++-
 python/mxnet/numpy/multiarray.py              |  96 +++++++++++-
 python/mxnet/numpy_dispatch_protocol.py       |   3 +
 python/mxnet/symbol/numpy/_symbol.py          |  24 ++-
 .../np_elemwise_broadcast_op_extended_sec.cc  |  56 +++++++
 src/operator/mshadow_op.h                     |  49 ++++++
 .../np_elemwise_broadcast_op_extended_sec.cc  | 142 ++++++++++++++++++
 .../np_elemwise_broadcast_op_extended_sec.cu  |  77 ++++++++++
 src/operator/operator_tune.cc                 |   4 +
 .../unittest/test_numpy_interoperability.py   |  24 +++
 tests/python/unittest/test_numpy_op.py        |  10 ++
 12 files changed, 558 insertions(+), 6 deletions(-)
 create mode 100644 src/api/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
 create mode 100644 src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
 create mode 100644 src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cu

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 45067a73c638..7b078a391720 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -110,6 +110,9 @@ def prepare_workloads():
     OpArgMngr.add_workload("random.logistic", loc=2, scale=2, size=(2,2))
     OpArgMngr.add_workload("random.gumbel", loc=2, scale=2, size=(2,2))
     OpArgMngr.add_workload("where", pool['2x3'], pool['2x3'], pool['2x1'])
+    OpArgMngr.add_workload("fmax", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("fmin", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("fmod", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("may_share_memory", pool['2x3'][:0], pool['2x3'][:1])
     OpArgMngr.add_workload("diag", pool['2x2'], k=1)
     OpArgMngr.add_workload("diagonal", pool['2x2x2'], offset=-1, axis1=0, axis2=1)
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index b04391125647..b77e4ac1159c 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -30,7 +30,8 @@
 
 
 __all__ = ['shape', 'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'empty_like', 'invert', 'delete',
-           'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'bitwise_not',
+           'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod',
+           'power', 'bitwise_not',
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'insert', 'fabs',
            'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul',
            'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram',
@@ -38,7 +39,7 @@
            'tensordot', 'eye', 'linspace', 'median',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'around', 'round', 'round_', 'flatnonzero',
+           'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'around', 'round', 'round_', 'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
@@ -1171,6 +1172,35 @@ def mod(x1, x2, out=None, **kwargs):
     return _api_internal.mod(x1, x2, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+@wrap_np_binary_func
+def fmod(x1, x2, out=None, **kwargs):
+    """
+    Return element-wise remainder of division.
+
+    Parameters
+    ----------
+    x1 : ndarray or scalar
+        Dividend array.
+
+    x2 : ndarray or scalar
+        Divisor array.
+
+    out : ndarray
+        A location into which the result is stored. If provided, it must have a shape
+        that the inputs broadcast to. If not provided or None, a freshly-allocated array
+        is returned.
+
+    Returns
+    -------
+    out : ndarray or scalar
+        This is a scalar if both x1 and x2 are scalars.
+    """
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        _np.fmod(x1, x2, out=out)
+    return _api_internal.fmod(x1, x2, out)
+
+
 @set_module('mxnet.ndarray.numpy')
 def delete(arr, obj, axis=None):
     """
@@ -4361,6 +4391,27 @@ def maximum(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.maximum, _np.maximum, _npi.maximum_scalar, None, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+@wrap_np_binary_func
+def fmax(x1, x2, out=None, **kwargs):
+    """
+    Returns element-wise maximum of the input arrays with broadcasting. (Ignores NaNs)
+
+    Parameters
+    ----------
+    x1, x2 : scalar or mxnet.numpy.ndarray
+        The arrays holding the elements to be compared. They must have the same shape,
+        or shapes that can be broadcast to a single shape.
+
+    Returns
+    -------
+    out : mxnet.numpy.ndarray or scalar
+        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars."""
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        _np.fmax(x1, x2, out=out)
+    return _api_internal.fmax(x1, x2, out)
+
+
 @set_module('mxnet.ndarray.numpy')
 @wrap_np_binary_func
 def minimum(x1, x2, out=None, **kwargs):
@@ -4380,6 +4431,27 @@ def minimum(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.minimum, _np.minimum, _npi.minimum_scalar, None, out)
 
 
+@set_module('mxnet.ndarray.numpy')
+@wrap_np_binary_func
+def fmin(x1, x2, out=None, **kwargs):
+    """
+    Returns element-wise minimum of the input arrays with broadcasting. (Ignores NaNs)
+
+    Parameters
+    ----------
+    x1, x2 : scalar or mxnet.numpy.ndarray
+        The arrays holding the elements to be compared. They must have the same shape,
+        or shapes that can be broadcast to a single shape.
+
+    Returns
+    -------
+    out : mxnet.numpy.ndarray or scalar
+        The minimum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars."""
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        _np.fmin(x1, x2, out=out)
+    return _api_internal.fmin(x1, x2, out)
+
+
 @set_module('mxnet.ndarray.numpy')
 def swapaxes(a, axis1, axis2):
     """Interchange two axes of an array.
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 38a9241138a1..e018a5b12d2a 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -53,7 +53,8 @@
 
 __all__ = ['ndarray', 'empty', 'empty_like', 'array', 'shape', 'median',
            'zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'all', 'any', 'broadcast_to',
-           'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'bitwise_not', 'delete',
+           'add', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod', 'power', 'bitwise_not',
+           'delete',
            'arctan2', 'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'invert',
            'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log',
            'degrees', 'log2', 'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'histogram',
@@ -61,7 +62,8 @@
            'sort', 'tensordot', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange',
            'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'flatnonzero',
            'concatenate', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'insert',
+           'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin',
+           'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'insert',
            'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman',
            'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',
            'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',
@@ -3136,6 +3138,38 @@ def mod(x1, x2, out=None, **kwargs):
     return _mx_nd_np.mod(x1, x2, out=out)
 
 
+@set_module('mxnet.numpy')
+@wrap_np_binary_func
+def fmod(x1, x2, out=None, **kwargs):
+    """
+    Return element-wise remainder of division.
+
+    Parameters
+    ----------
+    x1 : ndarray or scalar
+        Dividend array.
+
+    x2 : ndarray or scalar
+        Divisor array.
+
+    out : ndarray
+        A location into which the result is stored. If provided, it must have a shape
+        that the inputs broadcast to. If not provided or None, a freshly-allocated array
+        is returned.
+
+    Returns
+    -------
+    out : ndarray or scalar
+        This is a scalar if both x1 and x2 are scalars.
+
+    Examples
+    --------
+    >>> np.fmod(np.arange(7), 5)
+    array([0., 1., 2., 3., 4., 0., 1.])
+    """
+    return _mx_nd_np.fmod(x1, x2, out=out)
+
+
 @set_module('mxnet.numpy')
 @wrap_np_binary_func
 def matmul(a, b, out=None, **kwargs):
@@ -6174,6 +6208,35 @@ def maximum(x1, x2, out=None, **kwargs):
     return _mx_nd_np.maximum(x1, x2, out=out)
 
 
+@set_module('mxnet.numpy')
+@wrap_np_binary_func
+def fmax(x1, x2, out=None, **kwargs):
+    """
+    Returns element-wise maximum of the input arrays with broadcasting. (Ignores NaNs)
+
+    Parameters
+    ----------
+    x1, x2 : scalar or mxnet.numpy.ndarray
+        The arrays holding the elements to be compared. They must have the same shape,
+        or shapes that can be broadcast to a single shape.
+
+    Returns
+    -------
+    out : mxnet.numpy.ndarray or scalar
+        The maximum of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.
+
+    Examples
+    --------
+    >>> np.fmax(np.array([2, 3, 4]), np.array([1, 5, 2]))
+    array([2., 5., 4.])
+
+    >>> np.fmax(np.eye(2), np.array([0.5, 2])) # broadcasting
+    array([[1. , 2. ],
+           [0.5, 2. ]])
+    """
+    return _mx_nd_np.fmax(x1, x2, out=out)
+
+
 @set_module('mxnet.numpy')
 @wrap_np_binary_func
 def minimum(x1, x2, out=None, **kwargs):
@@ -6203,6 +6266,35 @@ def minimum(x1, x2, out=None, **kwargs):
     return _mx_nd_np.minimum(x1, x2, out=out)
 
 
+@set_module('mxnet.numpy')
+@wrap_np_binary_func
+def fmin(x1, x2, out=None, **kwargs):
+    """
+    Returns element-wise minimum of the input arrays with broadcasting. (Ignores NaNs)
+
+    Parameters
+    ----------
+    x1, x2 : scalar or mxnet.numpy.ndarray
+        The arrays holding the elements to be compared. They must have the same shape,
+        or shapes that can be broadcast to a single shape.
+
+    Returns
+    -------
+    out : mxnet.numpy.ndarray or scalar
+        The fmin of x1 and x2, element-wise. This is a scalar if both x1 and x2 are scalars.
+
+    Examples
+    --------
+    >>> np.fmin(np.array([2, 3, 4]), np.array([1, 5, 2]))
+    array([1., 3., 2.])
+
+    >>> np.fmin(np.eye(2), np.array([0.5, 2])) # broadcasting
+    array([[0.5, 0. ],
+           [0. , 1. ]])
+    """
+    return _mx_nd_np.fmin(x1, x2, out=out)
+
+
 @set_module('mxnet.numpy')
 def swapaxes(a, axis1, axis2):
     """Interchange two axes of an array.
diff --git a/python/mxnet/numpy_dispatch_protocol.py b/python/mxnet/numpy_dispatch_protocol.py
index c768dbc7a194..e693a00ea1a5 100644
--- a/python/mxnet/numpy_dispatch_protocol.py
+++ b/python/mxnet/numpy_dispatch_protocol.py
@@ -254,6 +254,7 @@ def _register_array_function():
     'negative',
     'power',
     'mod',
+    'fmod',
     'matmul',
     'absolute',
     'rint',
@@ -283,7 +284,9 @@ def _register_array_function():
     'arccosh',
     'arctanh',
     'maximum',
+    'fmax',
     'minimum',
+    'fmin',
     'ceil',
     'trunc',
     'floor',
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index df82c2598c60..06e432740242 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -36,14 +36,16 @@
     from builtins import slice as py_slice
 
 __all__ = ['zeros', 'zeros_like', 'ones', 'ones_like', 'full', 'full_like', 'empty_like', 'bitwise_not', 'invert',
-           'delete', 'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'power', 'arctan2',
+           'delete', 'add', 'broadcast_to', 'subtract', 'multiply', 'divide', 'mod', 'remainder', 'fmod',
+           'power', 'arctan2',
            'sin', 'cos', 'tan', 'sinh', 'cosh', 'tanh', 'log10', 'sqrt', 'cbrt', 'abs', 'absolute', 'fabs', 'exp',
            'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'log1p', 'matmul', 'median',
            'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram', 'insert',
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'sort', 'tensordot', 'eye', 'linspace',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
-           'average', 'mean', 'maximum', 'minimum', 'any', 'all', 'around', 'round', 'round_', 'flatnonzero',
+           'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'any', 'all', 'around', 'round', 'round_',
+           'flatnonzero',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'interp',
@@ -1582,6 +1584,12 @@ def mod(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.mod, _np.mod, _npi.mod_scalar, _npi.rmod_scalar, out)
 
 
+@set_module('mxnet.symbol.numpy')
+@wrap_np_binary_func
+def fmod(x1, x2, out=None, **kwargs):
+    return _ufunc_helper(x1, x2, _npi.fmod, _np.fmod, _npi.fmod_scalar, _npi.rfmod_scalar, out)
+
+
 @set_module('mxnet.symbol.numpy')
 @wrap_np_binary_func
 def remainder(x1, x2, out=None, **kwargs):
@@ -4096,12 +4104,24 @@ def maximum(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.maximum, _np.maximum, _npi.maximum_scalar, None, out)
 
 
+@set_module('mxnet.symbol.numpy')
+@wrap_np_binary_func
+def fmax(x1, x2, out=None, **kwargs):
+    return _ufunc_helper(x1, x2, _npi.fmax, _np.fmax, _npi.fmax_scalar, None, out)
+
+
 @set_module('mxnet.symbol.numpy')
 @wrap_np_binary_func
 def minimum(x1, x2, out=None, **kwargs):
     return _ufunc_helper(x1, x2, _npi.minimum, _np.minimum, _npi.minimum_scalar, None, out)
 
 
+@set_module('mxnet.symbol.numpy')
+@wrap_np_binary_func
+def fmin(x1, x2, out=None, **kwargs):
+    return _ufunc_helper(x1, x2, _npi.fmin, _np.fmin, _npi.fmin_scalar, None, out)
+
+
 @set_module('mxnet.symbol.numpy')
 def all(a, axis=None, out=None, keepdims=False):
     """
diff --git a/src/api/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc b/src/api/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
new file mode 100644
index 000000000000..248af4dd6e3e
--- /dev/null
+++ b/src/api/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_elemwise_broadcast_op_extended_sec.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../ufunc_helper.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.fmax")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_fmax");
+  const nnvm::Op* op_scalar = Op::Get("_npi_fmax_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.fmin")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_fmin");
+  const nnvm::Op* op_scalar = Op::Get("_npi_fmin_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.fmod")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_fmod");
+  const nnvm::Op* op_scalar = Op::Get("_npi_fmod_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rfmod_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+}  // namespace mxnet
diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index 2d4d49254676..4d9de29ce709 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -795,6 +795,27 @@ struct mod : public mxnet_op::tunable {
   }
 };
 
+struct fmod : public mxnet_op::tunable {
+  template<typename DType>
+  MSHADOW_XINLINE static DType Map(DType a, DType b) {
+    if (b == DType(0)) {
+      return DType(0);
+    } else {
+        return DType(::fmod(static_cast<double>(a), static_cast<double>(b)));
+    }
+  }
+};
+
+struct rfmod : public mxnet_op::tunable {
+  template<typename DType>
+  MSHADOW_XINLINE static DType Map(DType a, DType b) {
+    if (a == DType(0)) {
+      return DType(0);
+    } else  {
+      return DType(::fmod(static_cast<double>(b), static_cast<double>(a)));
+    }
+  }
+};
 
 template<>
 MSHADOW_XINLINE mshadow::half::half2_t mod::Map<mshadow::half::half2_t>
@@ -1132,6 +1153,20 @@ struct maximum : public mxnet_op::tunable {
   }
 };
 
+/*! \brief used for computing binary operator fmax */
+struct fmax : public mxnet_op::tunable {
+  template<typename DType>
+  MSHADOW_XINLINE static DType Map(DType a, DType b) {
+    if (IsNan(b)) {
+      return a;
+    } else if (IsNan(a)) {
+      return b;
+    } else {
+      return (a > b ? a : b);
+    }
+  }
+};
+
 /*! \brief used for computing binary operator minimum */
 struct minimum : public mxnet_op::tunable {
   template<typename DType>
@@ -1144,6 +1179,20 @@ struct minimum : public mxnet_op::tunable {
   }
 };
 
+/*! \brief used for computing binary operator fmin */
+struct fmin : public mxnet_op::tunable {
+  template<typename DType>
+  MSHADOW_XINLINE static DType Map(DType a, DType b) {
+    if (IsNan(b)) {
+      return a;
+    } else if (IsNan(a)) {
+      return b;
+    } else {
+      return (a < b ? a : b);
+    }
+  }
+};
+
 /*! \brief boolean any/all kernel that determines whether elem is NonZero */
 struct NonZero {
   template<typename DType>
diff --git a/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc b/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
new file mode 100644
index 000000000000..7455da139a14
--- /dev/null
+++ b/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cc
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file np_elemwise_broadcast_op_extended_sec.cc
+ * \brief CPU Implementation of extended functions for elementwise numpy binary broadcast operator. (Second extended file)
+ */
+
+#include "../../common/utils.h"
+#include "./np_elemwise_broadcast_op.h"
+
+namespace mxnet {
+namespace op {
+
+#define MXNET_OPERATOR_REGISTER_NP_BINARY_SCALAR(name)              \
+  NNVM_REGISTER_OP(name)                                            \
+  .set_num_inputs(1)                                                \
+  .set_num_outputs(1)                                               \
+  .set_attr_parser([](NodeAttrs* attrs) {                           \
+      attrs->parsed = std::stod(attrs->dict["scalar"]);             \
+    })                                                              \
+  .set_attr<mxnet::FInferShape>("FInferShape", ElemwiseShape<1, 1>) \
+  .set_attr<nnvm::FInferType>("FInferType", NumpyBinaryScalarType)  \
+  .set_attr<nnvm::FInplaceOption>("FInplaceOption",                 \
+    [](const NodeAttrs& attrs){                                     \
+      return std::vector<std::pair<int, int> >{{0, 0}};             \
+    })                                                              \
+  .add_argument("data", "NDArray-or-Symbol", "source input")        \
+  .add_argument("scalar", "float", "scalar input")
+
+MXNET_OPERATOR_REGISTER_BINARY_BROADCAST(_npi_fmax)
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastCompute<cpu, mshadow_op::fmax>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmax"});
+
+NNVM_REGISTER_OP(_backward_npi_fmax)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastBackwardUseIn<cpu, mshadow_op::ge,
+                                                                  mshadow_op::lt>);
+
+MXNET_OPERATOR_REGISTER_NP_BINARY_SCALAR(_npi_fmax_scalar)
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Compute<cpu, mshadow_op::fmax>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmax_scalar"});
+
+MXNET_OPERATOR_REGISTER_BINARY(_backward_npi_fmax_scalar)
+.add_argument("scalar", "float", "scalar value")
+.set_attr_parser([](NodeAttrs *attrs) { attrs->parsed = std::stod(attrs->dict["scalar"]); })
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Backward<cpu, mshadow_op::ge>);
+
+MXNET_OPERATOR_REGISTER_BINARY_BROADCAST(_npi_fmin)
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastCompute<cpu, mshadow_op::fmin>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmin"});
+
+NNVM_REGISTER_OP(_backward_npi_fmin)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastBackwardUseIn<cpu, mshadow_op::le,
+                                                                  mshadow_op::gt>);
+
+MXNET_OPERATOR_REGISTER_NP_BINARY_SCALAR(_npi_fmin_scalar)
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Compute<cpu, mshadow_op::fmin>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmin_scalar"});
+
+MXNET_OPERATOR_REGISTER_BINARY(_backward_npi_fmin_scalar)
+.add_argument("scalar", "float", "scalar value")
+.set_attr_parser([](NodeAttrs *attrs) { attrs->parsed = std::stod(attrs->dict["scalar"]); })
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Backward<cpu, mshadow_op::le>);
+
+MXNET_OPERATOR_REGISTER_BINARY_BROADCAST(_npi_fmod)
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastCompute<cpu, mshadow_op::fmod>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmod"});
+
+NNVM_REGISTER_OP(_backward_npi_fmod)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastBackwardUseIn<cpu, mshadow_op::mod_grad,
+                                                                  mshadow_op::mod_rgrad>);
+
+MXNET_OPERATOR_REGISTER_NP_BINARY_SCALAR(_npi_fmod_scalar)
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Compute<cpu, mshadow_op::fmod>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_fmod_scalar"});
+
+MXNET_OPERATOR_REGISTER_BINARY(_backward_npi_fmod_scalar)
+.add_argument("scalar", "float", "scalar value")
+.set_attr_parser([](NodeAttrs *attrs) { attrs->parsed = std::stod(attrs->dict["scalar"]); })
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Backward<cpu, mshadow_op::mod_grad>);
+
+MXNET_OPERATOR_REGISTER_NP_BINARY_SCALAR(_npi_rfmod_scalar)
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Compute<cpu, mshadow_op::rfmod>)
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_rfmod_scalar"});
+
+MXNET_OPERATOR_REGISTER_BINARY(_backward_npi_rfmod_scalar)
+.add_argument("scalar", "float", "scalar value")
+.set_attr_parser([](NodeAttrs *attrs) { attrs->parsed = std::stod(attrs->dict["scalar"]); })
+.set_attr<FCompute>("FCompute<cpu>", BinaryScalarOp::Backward<cpu, mshadow_op::rmod_grad>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cu b/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cu
new file mode 100644
index 000000000000..fa2f3bf080c7
--- /dev/null
+++ b/src/operator/numpy/np_elemwise_broadcast_op_extended_sec.cu
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ *  Copyright (c) 2019 by Contributors
+ * \file np_elemwise_broadcast_op_extended_sec.cu
+ * \brief GPU Implementation of extended functions for elementwise binary broadcast operator. (Second extended file)
+ */
+
+#include "./np_elemwise_broadcast_op.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_fmax)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, mshadow_op::fmax>);
+
+NNVM_REGISTER_OP(_backward_npi_fmax)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastBackwardUseIn<gpu, mshadow_op::ge,
+                                                                  mshadow_op::lt>);
+
+NNVM_REGISTER_OP(_npi_fmax_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Compute<gpu, mshadow_op::fmax>);
+
+NNVM_REGISTER_OP(_backward_npi_fmax_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Backward<gpu, mshadow_op::ge>);
+
+NNVM_REGISTER_OP(_npi_fmin)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, mshadow_op::fmin>);
+
+NNVM_REGISTER_OP(_backward_npi_fmin)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastBackwardUseIn<gpu, mshadow_op::le,
+                                                                  mshadow_op::gt>);
+
+NNVM_REGISTER_OP(_npi_fmin_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Compute<gpu, mshadow_op::fmin>);
+
+NNVM_REGISTER_OP(_backward_npi_fmin_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Backward<gpu, mshadow_op::le>);
+
+NNVM_REGISTER_OP(_npi_fmod)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, mshadow_op::fmod>);
+
+NNVM_REGISTER_OP(_backward_npi_fmod)
+.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastBackwardUseIn<gpu, mshadow_op::mod_grad,
+                                                                  mshadow_op::mod_rgrad>);
+
+NNVM_REGISTER_OP(_npi_fmod_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Compute<gpu, mshadow_op::fmod>);
+
+NNVM_REGISTER_OP(_backward_npi_fmod_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Backward<gpu, mshadow_op::mod_grad>);
+
+NNVM_REGISTER_OP(_npi_rfmod_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Compute<gpu, mshadow_op::rfmod>);
+
+NNVM_REGISTER_OP(_backward_npi_rfmod_scalar)
+.set_attr<FCompute>("FCompute<gpu>", BinaryScalarOp::Backward<gpu, mshadow_op::rmod_grad>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/operator_tune.cc b/src/operator/operator_tune.cc
index 0cc0dc92f884..b76e341b9fc6 100644
--- a/src/operator/operator_tune.cc
+++ b/src/operator/operator_tune.cc
@@ -345,6 +345,8 @@ IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::div_rgrad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::div_rgrad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::rdiv_grad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::mod);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::fmod);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::rfmod);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::mod_grad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::mod_rgrad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::rmod);  // NOLINT()
@@ -375,7 +377,9 @@ IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::gelu_grad); // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::prelu_grad); // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::elu_grad); // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::maximum);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::fmax);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::minimum);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::fmin);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::hypot);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::hypot_grad_left);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::hypot_grad_left);  // NOLINT()
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index e88bf88a10ab..79a45f5e46f6 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -1499,6 +1499,13 @@ def _add_workload_mod(array_pool):
     OpArgMngr.add_workload('mod', array_pool['4x1'], array_pool['1x1x0'])
 
 
+def _add_workload_fmod(array_pool):
+    OpArgMngr.add_workload('fmod', array_pool['4x1'], array_pool['1x2'])
+    OpArgMngr.add_workload('fmod', array_pool['4x1'], 2)
+    OpArgMngr.add_workload('fmod', 2, array_pool['4x1'])
+    OpArgMngr.add_workload('fmod', array_pool['4x1'], array_pool['1x1x0'])
+
+
 def _add_workload_remainder():
     # test remainder basic
     OpArgMngr.add_workload('remainder', np.array([0, 1, 2, 4, 2], dtype=np.float16),
@@ -1565,6 +1572,13 @@ def _add_workload_maximum(array_pool):
     OpArgMngr.add_workload('maximum', array_pool['4x1'], array_pool['1x1x0'])
 
 
+def _add_workload_fmax(array_pool):
+    OpArgMngr.add_workload('fmax', array_pool['4x1'], array_pool['1x2'])
+    OpArgMngr.add_workload('fmax', array_pool['4x1'], 2)
+    OpArgMngr.add_workload('fmax', 2, array_pool['4x1'])
+    OpArgMngr.add_workload('fmax', array_pool['4x1'], array_pool['1x1x0'])
+
+
 def _add_workload_minimum(array_pool):
     OpArgMngr.add_workload('minimum', array_pool['4x1'], array_pool['1x2'])
     OpArgMngr.add_workload('minimum', array_pool['4x1'], 2)
@@ -1572,6 +1586,13 @@ def _add_workload_minimum(array_pool):
     OpArgMngr.add_workload('minimum', array_pool['4x1'], array_pool['1x1x0'])
 
 
+def _add_workload_fmin(array_pool):
+    OpArgMngr.add_workload('fmin', array_pool['4x1'], array_pool['1x2'])
+    OpArgMngr.add_workload('fmin', array_pool['4x1'], 2)
+    OpArgMngr.add_workload('fmin', 2, array_pool['4x1'])
+    OpArgMngr.add_workload('fmin', array_pool['4x1'], array_pool['1x1x0'])
+
+
 def _add_workload_negative(array_pool):
     OpArgMngr.add_workload('negative', array_pool['4x1'])
 
@@ -2920,9 +2941,12 @@ def _prepare_workloads():
     _add_workload_multiply(array_pool)
     _add_workload_power(array_pool)
     _add_workload_mod(array_pool)
+    _add_workload_fmod(array_pool)
     _add_workload_remainder()
     _add_workload_maximum(array_pool)
+    _add_workload_fmax(array_pool)
     _add_workload_minimum(array_pool)
+    _add_workload_fmin(array_pool)
     _add_workload_negative(array_pool)
     _add_workload_absolute(array_pool)
     _add_workload_sign(array_pool)
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 9e4b777c77d7..3e4562feb0b9 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -2492,6 +2492,12 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
                 [lambda y, x1, x2: -_np.floor(x1 / x2),
                  lambda y, x1, x2: _np.zeros(y.shape)],
                 [[_np.float16, _np.float32, _np.float64], [_np.int32]]),
+        'fmod': (1.0, 10.0,
+                [lambda y, x1, x2: _np.ones(y.shape),
+                 lambda y, x1, x2: _np.zeros(y.shape)],
+                [lambda y, x1, x2: -_np.floor(x1 / x2),
+                 lambda y, x1, x2: _np.zeros(y.shape)],
+                [[_np.float16, _np.float32, _np.float64], [_np.int32]]),
         'remainder': (1.0, 10.0,
                       [lambda y, x1, x2: _np.ones(y.shape),
                        lambda y, x1, x2: _np.zeros(y.shape)],
@@ -2506,8 +2512,12 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
         'bitwise_or': (-100, 100, [None], None, [[_np.int32]]),
         'maximum': (-1, 1, [lambda y, x1, x2: _np.ones(y.shape) * (x1 >= x2)],
                            [lambda y, x1, x2: _np.ones(y.shape) * (x1 < x2)]),
+        'fmax': (-1, 1, [lambda y, x1, x2: _np.ones(y.shape) * (x1 >= x2)],
+                        [lambda y, x1, x2: _np.ones(y.shape) * (x1 < x2)]),
         'minimum': (-1, 1, [lambda y, x1, x2: _np.ones(y.shape) * (x1 <= x2)],
                            [lambda y, x1, x2: _np.ones(y.shape) * (x1 > x2)]),
+        'fmin': (-1, 1, [lambda y, x1, x2: _np.ones(y.shape) * (x1 <= x2)],
+                        [lambda y, x1, x2: _np.ones(y.shape) * (x1 > x2)]),
         'copysign': (-1, 1,
                      [lambda y, x1, x2: _np.ones(y.shape) * (((x1 * x2) >= 0).astype(_np.float32) - ((x1 * x2) < 0).astype(_np.float32))],
                      [lambda y, x1, x2: _np.zeros(y.shape)]),

From 673d3b3c0d239e015f821b667b830e1bd05c54a2 Mon Sep 17 00:00:00 2001
From: Yijun Chen <chenyijun0902@gmail.com>
Date: Fri, 10 Apr 2020 14:01:46 +0800
Subject: [PATCH 38/44] add: numpy op tril_indices (#17904)

---
 python/mxnet/ndarray/numpy/_op.py      |  84 ++++++++++++++++++++-
 python/mxnet/numpy/multiarray.py       |  80 +++++++++++++++++++-
 python/mxnet/symbol/numpy/_symbol.py   |  85 ++++++++++++++++++++-
 src/operator/numpy/np_matrix_op-inl.h  | 100 +++++++++++++++++++++++++
 src/operator/numpy/np_matrix_op.cc     |  60 +++++++++++++++
 src/operator/numpy/np_matrix_op.cu     |   3 +
 tests/python/unittest/test_numpy_op.py |  37 +++++++++
 7 files changed, 446 insertions(+), 3 deletions(-)

diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index b77e4ac1159c..eeef0b883ee4 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -36,7 +36,7 @@
            'absolute', 'exp', 'expm1', 'arcsin', 'arccos', 'arctan', 'sign', 'log', 'degrees', 'log2', 'matmul',
            'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'fix', 'ceil', 'floor', 'histogram',
            'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'argsort', 'all', 'any', 'sort',
-           'tensordot', 'eye', 'linspace', 'median',
+           'tensordot', 'eye', 'linspace', 'median', 'tril_indices',
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
            'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'around', 'round', 'round_', 'flatnonzero',
@@ -4530,6 +4530,88 @@ def clip(a, a_min, a_max, out=None):
     return _npi.clip(a, a_min, a_max, out=out)
 
 
+@set_module('mxnet.ndarray.numpy')
+def tril_indices(n, k=0, m=None):
+    """
+    Return the indices for the lower-triangle of an (n, m) array.
+
+    Parameters
+    ----------
+    n : int
+        The row dimension of the arrays for which the returned
+        indices will be valid.
+    k : int, optional
+        Diagonal offset (see `tril` for details).
+    m : int, optional
+        .. versionadded:: 1.9.0
+
+        The column dimension of the arrays for which the returned
+        arrays will be valid.
+        By default `m` is taken equal to `n`.
+
+    Returns
+    -------
+    inds : tuple of arrays
+        The indices for the triangle. The returned tuple contains two arrays,
+        each with the indices along one dimension of the array.
+
+    See also
+    --------
+    triu_indices : similar function, for upper-triangular.
+    mask_indices : generic function accepting an arbitrary mask function.
+    tril, triu
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
+    Compute two different sets of indices to access 4x4 arrays, one for the
+    lower triangular part starting at the main diagonal, and one starting two
+    diagonals further right:
+
+    >>> il1 = np.tril_indices(4)
+    >>> il2 = np.tril_indices(4, 2)
+
+    Here is how they can be used with a sample array:
+
+    >>> a = np.arange(16).reshape(4, 4)
+    >>> a
+    array([[ 0,  1,  2,  3],
+           [ 4,  5,  6,  7],
+           [ 8,  9, 10, 11],
+           [12, 13, 14, 15]])
+
+    Both for indexing:
+
+    >>> a[il1]
+    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])
+
+    And for assigning values:
+
+    >>> a[il1] = -1
+    >>> a
+    array([[-1,  1,  2,  3],
+           [-1, -1,  6,  7],
+           [-1, -1, -1, 11],
+           [-1, -1, -1, -1]])
+
+    These cover almost the whole array (two diagonals right of the main one):
+
+    >>> a[il2] = -10
+    >>> a
+    array([[-10, -10, -10,   3],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10]])
+
+    """
+    if m is None:
+        m = n
+    return tuple(_npi.tril_indices(n, k, m))
+
+
 @set_module('mxnet.ndarray.numpy')
 def argmax(a, axis=None, out=None):
     r"""
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index e018a5b12d2a..f250d6a73856 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -60,7 +60,7 @@
            'degrees', 'log2', 'log1p', 'rint', 'radians', 'reciprocal', 'square', 'negative', 'histogram',
            'fix', 'ceil', 'floor', 'trunc', 'logical_not', 'arcsinh', 'arccosh', 'arctanh', 'append', 'argsort',
            'sort', 'tensordot', 'eye', 'linspace', 'logspace', 'expand_dims', 'tile', 'arange',
-           'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'flatnonzero',
+           'array_split', 'split', 'hsplit', 'vsplit', 'dsplit', 'flatnonzero', 'tril_indices',
            'concatenate', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
            'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'insert',
@@ -5516,6 +5516,84 @@ def tril(m, k=0):
     return _mx_nd_np.tril(m, k)
 
 
+@set_module('mxnet.numpy')
+def tril_indices(n, k=0, m=None):
+    """
+    Return the indices for the lower-triangle of an (n, m) array.
+
+    Parameters
+    ----------
+    n : int
+        The row dimension of the arrays for which the returned
+        indices will be valid.
+    k : int, optional
+        Diagonal offset (see `tril` for details).
+    m : int, optional
+        .. versionadded:: 1.9.0
+
+        The column dimension of the arrays for which the returned
+        arrays will be valid.
+        By default `m` is taken equal to `n`.
+
+    Returns
+    -------
+    inds : tuple of arrays
+        The indices for the triangle. The returned tuple contains two arrays,
+        each with the indices along one dimension of the array.
+
+    See also
+    --------
+    triu_indices : similar function, for upper-triangular.
+    mask_indices : generic function accepting an arbitrary mask function.
+    tril, triu
+
+    Examples
+    --------
+    Compute two different sets of indices to access 4x4 arrays, one for the
+    lower triangular part starting at the main diagonal, and one starting two
+    diagonals further right:
+
+    >>> il1 = np.tril_indices(4)
+    >>> il2 = np.tril_indices(4, 2)
+
+    Here is how they can be used with a sample array:
+
+    >>> a = np.arange(16).reshape(4, 4)
+    >>> a
+    array([[ 0,  1,  2,  3],
+           [ 4,  5,  6,  7],
+           [ 8,  9, 10, 11],
+           [12, 13, 14, 15]])
+
+    Both for indexing:
+
+    >>> a[il1]
+    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])
+
+    And for assigning values:
+
+    >>> a[il1] = -1
+    >>> a
+    array([[-1,  1,  2,  3],
+           [-1, -1,  6,  7],
+           [-1, -1, -1, 11],
+           [-1, -1, -1, -1]])
+
+    These cover almost the whole array (two diagonals right of the main one):
+
+    >>> a[il2] = -10
+    >>> a
+    array([[-10, -10, -10,   3],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10]])
+
+    """
+    if m is None:
+        m = n
+    return tuple(_mx_nd_np.tril_indices(n, k, m))
+
+
 # pylint: disable=redefined-outer-name
 @set_module('mxnet.numpy')
 def arange(start, stop=None, step=1, dtype=None, ctx=None):
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 06e432740242..5cebb2c8fe52 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -45,7 +45,7 @@
            'logspace', 'expand_dims', 'tile', 'arange', 'array_split', 'split', 'hsplit', 'vsplit', 'dsplit',
            'concatenate', 'append', 'stack', 'vstack', 'row_stack', 'column_stack', 'hstack', 'dstack',
            'average', 'mean', 'maximum', 'fmax', 'minimum', 'fmin', 'any', 'all', 'around', 'round', 'round_',
-           'flatnonzero',
+           'flatnonzero', 'tril_indices',
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'interp',
@@ -2160,6 +2160,89 @@ def tril(m, k=0):
     return _npi.tril(m, k)
 
 
+@set_module('mxnet.symbol.numpy')
+def tril_indices(n, k=0, m=None):
+    """
+    Return the indices for the lower-triangle of an (n, m) array.
+
+    Parameters
+    ----------
+    n : int
+        The row dimension of the arrays for which the returned
+        indices will be valid.
+    k : int, optional
+        Diagonal offset (see `tril` for details).
+    m : int, optional
+        .. versionadded:: 1.9.0
+
+        The column dimension of the arrays for which the returned
+        arrays will be valid.
+        By default `m` is taken equal to `n`.
+
+
+    Returns
+    -------
+    inds : tuple of _Symbol
+        The indices for the triangle. The returned tuple contains two arrays,
+        each with the indices along one dimension of the array.
+
+    See also
+    --------
+    triu_indices : similar function, for upper-triangular.
+    mask_indices : generic function accepting an arbitrary mask function.
+    tril, triu
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    Examples
+    --------
+    Compute two different sets of indices to access 4x4 arrays, one for the
+    lower triangular part starting at the main diagonal, and one starting two
+    diagonals further right:
+
+    >>> il1 = np.tril_indices(4)
+    >>> il2 = np.tril_indices(4, 2)
+
+    Here is how they can be used with a sample array:
+
+    >>> a = np.arange(16).reshape(4, 4)
+    >>> a
+    array([[ 0,  1,  2,  3],
+           [ 4,  5,  6,  7],
+           [ 8,  9, 10, 11],
+           [12, 13, 14, 15]])
+
+    Both for indexing:
+
+    >>> a[il1]
+    array([ 0,  4,  5,  8,  9, 10, 12, 13, 14, 15])
+
+    And for assigning values:
+
+    >>> a[il1] = -1
+    >>> a
+    array([[-1,  1,  2,  3],
+           [-1, -1,  6,  7],
+           [-1, -1, -1, 11],
+           [-1, -1, -1, -1]])
+
+    These cover almost the whole array (two diagonals right of the main one):
+
+    >>> a[il2] = -10
+    >>> a
+    array([[-10, -10, -10,   3],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10],
+           [-10, -10, -10, -10]])
+
+    """
+    if m is None:
+        m = n
+    return _npi.tril_indices(n, k, m)
+
+
 def _unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):
     """Helper function for unary operators.
 
diff --git a/src/operator/numpy/np_matrix_op-inl.h b/src/operator/numpy/np_matrix_op-inl.h
index 223431198c82..f75ca82d323d 100644
--- a/src/operator/numpy/np_matrix_op-inl.h
+++ b/src/operator/numpy/np_matrix_op-inl.h
@@ -287,6 +287,106 @@ void NumpyVstackBackward(const nnvm::NodeAttrs& attrs,
   });
 }
 
+struct NumpyTrilindicesParam : public dmlc::Parameter<NumpyTrilindicesParam> {
+  int n;
+  int k;
+  int m;
+  DMLC_DECLARE_PARAMETER(NumpyTrilindicesParam) {
+    DMLC_DECLARE_FIELD(n)
+      .describe("The row dimension of the arrays for which"
+                "the returned indices will be valid.");
+    DMLC_DECLARE_FIELD(k)
+      .set_default(0)
+      .describe("Diagonal offset");
+    DMLC_DECLARE_FIELD(m)
+      .describe("The column dimension of the arrays for "
+                "which the returned arrays will be valid."
+                "By default m is taken equal to n.");
+  }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream n_s, k_s, m_s;
+    n_s << n;
+    k_s << k;
+    m_s << m;
+    (*dict)["n"] = n_s.str();
+    (*dict)["k"] = k_s.str();
+    (*dict)["m"] = m_s.str();
+  }
+};
+
+template<int req>
+struct TrilindicesOpForwardImpl {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(int i, DType* out_data0, DType* out_data1,
+                                  int* data, int length) {
+    KERNEL_ASSIGN(out_data0[i], req, data[i]);
+    KERNEL_ASSIGN(out_data1[i], req, data[i + length]);
+  }
+};
+
+template<typename xpu>
+void TrilindicesOpForward(const nnvm::NodeAttrs& attrs,
+                          const OpContext& ctx,
+                          const std::vector<TBlob>& inputs,
+                          const std::vector<OpReqType>& req,
+                          const std::vector<TBlob>& outputs) {
+  using namespace mxnet_op;
+  using namespace mshadow;
+  CHECK_EQ(inputs.size(), 0U);
+  CHECK_EQ(outputs.size(), 2U);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const NumpyTrilindicesParam& param =
+    nnvm::get<NumpyTrilindicesParam>(attrs.parsed);
+
+  const TBlob& out_data0 = outputs[0];
+  const TBlob& out_data1 = outputs[1];
+
+  CHECK_EQ(out_data0.shape_[0], out_data1.shape_[0]);
+  int length = out_data0.shape_[0];
+
+  std::vector<int> indices_cpu(2 * length, 0);
+  size_t total_temp_size = 2 * length * sizeof(int);
+  Tensor<xpu, 1, char> temp_space =
+    ctx.requested[0].get_space_typed<xpu, 1, char>(Shape1(total_temp_size), s);
+  int* indices = reinterpret_cast<int*>(temp_space.dptr_);
+
+  int n = param.n;
+  int m = param.m;
+  int k = param.k;
+
+  int end = k;
+  int idx = 0;
+  for (int i = 0; i < n; i++) {
+    for (int j = 0; j <= std::min(end, m - 1); j++) {
+      indices_cpu[idx] = i;
+      indices_cpu[idx + length] = j;
+      idx++;
+    }
+    end++;
+  }
+
+  if (ctx.run_ctx.ctx.dev_mask() == gpu::kDevMask) {
+  #if MXNET_USE_CUDA
+    cudaMemcpyAsync(indices, indices_cpu.data(),
+                    indices_cpu.size() * sizeof(int),
+                    cudaMemcpyHostToDevice,
+                    Stream<gpu>::GetStream(ctx.get_stream<gpu>()));
+  #else
+    LOG(FATAL) << "Illegal attempt to use GPU in a CPU-only build";
+  #endif
+  } else {
+    std::memcpy(indices, indices_cpu.data(), indices_cpu.size() * sizeof(int));
+  }
+
+  MSHADOW_IDX_TYPE_SWITCH(out_data0.type_flag_, DType, {
+    MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+      Kernel<TrilindicesOpForwardImpl<req_type>, xpu>::Launch(
+          s, length, out_data0.dptr<DType>(), out_data1.dptr<DType>(),
+           indices, length);
+    });
+  });
+}
+
 struct NumpyRollParam : public dmlc::Parameter<NumpyRollParam> {
   dmlc::optional<mxnet::TShape> shift;
   dmlc::optional<mxnet::TShape> axis;
diff --git a/src/operator/numpy/np_matrix_op.cc b/src/operator/numpy/np_matrix_op.cc
index 9e2d91a31815..afed9a1a620b 100644
--- a/src/operator/numpy/np_matrix_op.cc
+++ b/src/operator/numpy/np_matrix_op.cc
@@ -1116,6 +1116,66 @@ NNVM_REGISTER_OP(_backward_np_dstack)
 .set_attr<nnvm::TIsBackward>("TIsBackward", true)
 .set_attr<FCompute>("FCompute<cpu>", DStackGradCompute<cpu>);
 
+DMLC_REGISTER_PARAMETER(NumpyTrilindicesParam);
+
+inline bool TrilindicesOpType(const nnvm::NodeAttrs& attrs,
+                                std::vector<int> *in_attrs,
+                                std::vector<int> *out_attrs) {
+  CHECK_EQ(in_attrs->size(), 0U);
+  CHECK_EQ(out_attrs->size(), 2U);
+
+  TYPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::kInt64);
+  TYPE_ASSIGN_CHECK(*out_attrs, 1, mshadow::kInt64);
+
+  return true;
+}
+
+inline bool TrilindicesOpShape(const nnvm::NodeAttrs& attrs,
+                               mxnet::ShapeVector* in_attrs,
+                               mxnet::ShapeVector* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 0U);
+  CHECK_EQ(out_attrs->size(), 2U);
+
+  const NumpyTrilindicesParam& param =
+    nnvm::get<NumpyTrilindicesParam>(attrs.parsed);
+
+  int n = param.n;
+  int m = param.m;
+  int k = param.k;
+
+  int length = 0;
+  int end = k;
+  for (int i = 0; i < n; i++) {
+    int tmpCount = 0;
+    for (int j = 0; j <= std::min(end, m - 1); j++) {
+      tmpCount++;
+    }
+    length += tmpCount;
+    end++;
+  }
+
+  mxnet::TShape oshape;
+  oshape = mxnet::TShape(1, length);
+
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape);
+  SHAPE_ASSIGN_CHECK(*out_attrs, 1, oshape);
+
+  return shape_is_known(out_attrs->at(0)) && shape_is_known(out_attrs->at(1));
+}
+
+NNVM_REGISTER_OP(_npi_tril_indices)
+.set_attr_parser(ParamParser<NumpyTrilindicesParam>)
+.set_num_inputs(0)
+.set_num_outputs(2)
+.set_attr<mxnet::FInferShape>("FInferShape", TrilindicesOpShape)
+.set_attr<nnvm::FInferType>("FInferType", TrilindicesOpType)
+.set_attr<FCompute>("FCompute<cpu>", TrilindicesOpForward<cpu>)
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& n) {
+     return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.add_arguments(NumpyTrilindicesParam::__FIELDS__());
+
 inline bool NumpyRollShape(const nnvm::NodeAttrs& attrs,
                            mxnet::ShapeVector *in_attrs,
                            mxnet::ShapeVector *out_attrs) {
diff --git a/src/operator/numpy/np_matrix_op.cu b/src/operator/numpy/np_matrix_op.cu
index 4871629d98e5..f919bb5f9c10 100644
--- a/src/operator/numpy/np_matrix_op.cu
+++ b/src/operator/numpy/np_matrix_op.cu
@@ -71,6 +71,9 @@ NNVM_REGISTER_OP(_npi_column_stack)
 NNVM_REGISTER_OP(_backward_np_column_stack)
 .set_attr<FCompute>("FCompute<gpu>", NumpyColumnStackBackward<gpu>);
 
+NNVM_REGISTER_OP(_npi_tril_indices)
+.set_attr<FCompute>("FCompute<gpu>", TrilindicesOpForward<gpu>);
+
 NNVM_REGISTER_OP(_npi_roll)
 .set_attr<FCompute>("FCompute<gpu>", NumpyRollCompute<gpu>);
 
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 3e4562feb0b9..ba6f1baef545 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -7149,6 +7149,43 @@ def test_np_builtin_op_signature():
         assert str(op.__signature__) == str(inspect.signature(_op_from_doc))
 
 
+@with_seed()
+@use_np
+def test_np_tril_indices():
+    class TestTrilindices(HybridBlock):
+        def __init__(self, n, k=0, m=None):
+            super(TestTrilindices, self).__init__()
+            self._n = n;
+            self._k = k;
+            if m is None:
+                m = n
+            self._m = m
+        
+        def hybrid_forward(self, F, x, *args, **kwargs):
+            return x, F.np.tril_indices(n=self._n, k=self._k, m=self._m)
+    
+    for n in _np.random.random_integers(-10, 50, 2):
+        for k in _np.random.random_integers(-50, 50, 2):
+            for m in _np.random.random_integers(-10, 50, 2):
+                np_out = _np.tril_indices(n, k, m)
+                for hybridize in [True, False]:
+                    # dummy nparray for hybridize
+                    x = np.ones((1,1))
+                    test_trilindices = TestTrilindices(n, k, m)
+                    if hybridize:
+                        test_trilindices.hybridize()
+                    mx_out = test_trilindices(x)[1]
+                    assert len(mx_out) == 2
+                    assert same(mx_out[0], np_out[0])
+                    assert same(mx_out[1], np_out[1])
+                    if n > 0 and m > 0 and hybridize is False:
+                        np_data = _np.arange(n*m).reshape(n, m)
+                        mx_data = np.array(np_data)
+                        np_data[np_out] = -10
+                        mx_data[mx_out] = -10
+                        assert same(np_data, mx_data.asnumpy())
+                        
+
 @with_seed()
 @use_np
 def test_np_moveaxis():

From 5ba7a773cf8f2e9c6133d89198b2fde9b0ef8359 Mon Sep 17 00:00:00 2001
From: vexilligera <vexilligera@gmail.com>
Date: Tue, 14 Apr 2020 23:33:41 +0000
Subject: [PATCH 39/44] [NumPy] Add NumPy support for triu (#17614)

* triu

* rebase

* fix ci

* merge

* triu new ffi

* cpplint

* cpplint

* ffi benchmark

* fix style

* merge

* fix conflict

Co-authored-by: Ubuntu <ubuntu@ip-172-31-8-137.us-east-2.compute.internal>
Co-authored-by: Hao Jin <hjjn.amzn@gmail.com>
---
 benchmark/python/ffi/benchmark_ffi.py         |   1 +
 python/mxnet/ndarray/numpy/_op.py             |  27 +-
 python/mxnet/numpy/multiarray.py              |  27 +-
 python/mxnet/symbol/numpy/_symbol.py          |  28 +-
 src/api/operator/numpy/np_triu_op.cc          |  50 ++++
 src/operator/numpy/np_triu_op-inl.h           | 241 ++++++++++++++++++
 src/operator/numpy/np_triu_op.cc              |  61 +++++
 src/operator/numpy/np_triu_op.cu              |  38 +++
 .../unittest/test_numpy_interoperability.py   |  17 ++
 tests/python/unittest/test_numpy_op.py        |  62 +++++
 10 files changed, 549 insertions(+), 3 deletions(-)
 create mode 100644 src/api/operator/numpy/np_triu_op.cc
 create mode 100644 src/operator/numpy/np_triu_op-inl.h
 create mode 100644 src/operator/numpy/np_triu_op.cc
 create mode 100644 src/operator/numpy/np_triu_op.cu

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 7b078a391720..bc55164d9663 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -128,6 +128,7 @@ def prepare_workloads():
                            out=dnp.array([False, False], dtype=bool), keepdims=False)
     OpArgMngr.add_workload("roll", pool["2x2"], 1, axis=0)
     OpArgMngr.add_workload("rot90", pool["2x2"], 2)
+    OpArgMngr.add_workload("triu", pool['3x3'])
     OpArgMngr.add_workload("array_split", pool['2x2'], 2, axis=1)
     OpArgMngr.add_workload("vsplit", pool['2x2'], 2)
     OpArgMngr.add_workload("hsplit", pool['2x2'], 2)
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index eeef0b883ee4..c1ce9092904a 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -43,7 +43,7 @@
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm',
-           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
+           'tril', 'triu', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'interp',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
@@ -2070,6 +2070,31 @@ def tril(m, k=0):
     return _api_internal.tril(m, k)
 
 
+@set_module('mxnet.ndarray.numpy')
+def triu(m, k=0):
+    r"""
+    Upper triangle of an array.
+
+    Return a copy of a matrix with the elements below the `k`-th diagonal
+    zeroed.
+
+    Please refer to the documentation for `tril` for further details.
+
+    See Also
+    --------
+    tril : lower triangle of an array
+
+    Examples
+    --------
+    >>> np.triu(np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]]), -1)
+    array([[ 1,  2,  3],
+           [ 4,  5,  6],
+           [ 0,  8,  9],
+           [ 0,  0, 12]])
+    """
+    return _api_internal.triu(m, k)
+
+
 def _unary_func_helper(x, fn_array, fn_scalar, out=None, **kwargs):
     """Helper function for unary operators.
 
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index f250d6a73856..57807f78e388 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -67,7 +67,7 @@
            'indices', 'copysign', 'ravel', 'unravel_index', 'diag_indices_from', 'hanning', 'hamming', 'blackman',
            'flip', 'flipud', 'fliplr', 'around', 'round', 'round_', 'arctan2', 'hypot',
            'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad',
-           'unique', 'lcm', 'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
+           'unique', 'lcm', 'tril', 'triu', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'interp',
            'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum', 'true_divide', 'nonzero',
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
@@ -5595,6 +5595,31 @@ def tril_indices(n, k=0, m=None):
 
 
 # pylint: disable=redefined-outer-name
+@set_module('mxnet.numpy')
+def triu(m, k=0):
+    r"""
+    Upper triangle of an array.
+
+    Return a copy of a matrix with the elements below the `k`-th diagonal
+    zeroed.
+
+    Please refer to the documentation for `tril` for further details.
+
+    See Also
+    --------
+    tril : lower triangle of an array
+
+    Examples
+    --------
+    >>> np.triu(np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]]), -1)
+    array([[ 1,  2,  3],
+           [ 4,  5,  6],
+           [ 0,  8,  9],
+           [ 0,  0, 12]])
+    """
+    return _mx_nd_np.triu(m, k)
+
+
 @set_module('mxnet.numpy')
 def arange(start, stop=None, step=1, dtype=None, ctx=None):
     """Return evenly spaced values within a given interval.
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index 5cebb2c8fe52..aae0ed21efc1 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -49,7 +49,7 @@
            'swapaxes', 'clip', 'argmax', 'argmin', 'std', 'var', 'indices', 'copysign', 'ravel', 'unravel_index',
            'diag_indices_from', 'hanning', 'hamming', 'blackman', 'flip', 'flipud', 'fliplr',
            'hypot', 'bitwise_and', 'bitwise_xor', 'bitwise_or', 'rad2deg', 'deg2rad', 'unique', 'lcm', 'interp',
-           'tril', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
+           'tril', 'triu', 'identity', 'take', 'ldexp', 'vdot', 'inner', 'outer', 'kron',
            'equal', 'not_equal', 'greater', 'less', 'greater_equal', 'less_equal', 'roll', 'rot90', 'einsum',
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
@@ -2161,6 +2161,32 @@ def tril(m, k=0):
 
 
 @set_module('mxnet.symbol.numpy')
+def triu(m, k=0):
+    r"""
+    Upper triangle of an array.
+
+    Return a copy of an array with elements under the `k`-th diagonal zeroed.
+
+    Parameters
+    ----------
+    m : _Symbol, shape (M, N)
+        Input array.
+    k : int, optional
+        Diagonal under which to zero elements.  `k = 0` (the default) is the
+        main diagonal, `k < 0` is below it and `k > 0` is under.
+
+    Returns
+    -------
+    triu : _Symbol, shape (M, N)
+        Upper triangle of `m`, of same shape and data-type as `m`.
+
+    See Also
+    --------
+    tril : same thing, only for the lower triangle
+    """
+    return _npi.triu(m, k)
+
+
 def tril_indices(n, k=0, m=None):
     """
     Return the indices for the lower-triangle of an (n, m) array.
diff --git a/src/api/operator/numpy/np_triu_op.cc b/src/api/operator/numpy/np_triu_op.cc
new file mode 100644
index 000000000000..e42169aca43b
--- /dev/null
+++ b/src/api/operator/numpy/np_triu_op.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_cumsum.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_triu_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_triu_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.triu")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  op::TriuParam param;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = Op::Get("_npi_triu");
+  // inputs
+  param.k = args[1].operator int();
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+
+  attrs.op = op;
+  attrs.parsed = param;
+  SetAttrDict<op::TriuParam>(&attrs);
+
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, nullptr);
+  *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_triu_op-inl.h b/src/operator/numpy/np_triu_op-inl.h
new file mode 100644
index 000000000000..17a484f26efb
--- /dev/null
+++ b/src/operator/numpy/np_triu_op-inl.h
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_triu_op-inl.h
+ * \brief Function definition of triu (upper triangle of an array) op
+ */
+
+#ifndef MXNET_OPERATOR_NUMPY_NP_TRIU_OP_INL_H_
+#define MXNET_OPERATOR_NUMPY_NP_TRIU_OP_INL_H_
+
+#include <dmlc/parameter.h>
+#include <vector>
+#include <string>
+#include <unordered_map>
+#include <sstream>
+#include <algorithm>
+#include "../mxnet_op.h"
+#include "../operator_common.h"
+#include "../elemwise_op_common.h"
+
+namespace mxnet {
+namespace op {
+
+struct TriuParam : public dmlc::Parameter<TriuParam> {
+  int k;
+  DMLC_DECLARE_PARAMETER(TriuParam) {
+    DMLC_DECLARE_FIELD(k)
+      .set_default(0)
+      .describe("Diagonal in question. The default is 0. "
+                "Use k>0 for diagonals above the main diagonal, "
+                "and k<0 for diagonals below the main diagonal. "
+                "If input has shape (S0 S1) k must be between -S0 and S1.");
+  }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream k_s;
+    k_s << k;
+    (*dict)["k"] = k_s.str();
+  }
+};
+
+inline bool TriuOpShape(const nnvm::NodeAttrs& attrs,
+                        mxnet::ShapeVector* in_attrs,
+                        mxnet::ShapeVector* out_attrs) {
+  CHECK_EQ(in_attrs->size(), 1U);
+  CHECK_EQ(out_attrs->size(), 1U);
+
+  const mxnet::TShape& ishape = (*in_attrs)[0];
+  mxnet::TShape oshape;
+
+  if (!mxnet::ndim_is_known(ishape)) {
+    return false;
+  }
+
+  if (ishape.ndim() == 1) {
+    auto s = ishape[0];
+    oshape = mxnet::TShape({s, s});
+  } else {
+    oshape = ishape;
+  }
+
+  if (shape_is_none(oshape)) {
+    LOG(FATAL) << "Diagonal does not exist.";
+  }
+  SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape);
+
+  return shape_is_known(out_attrs->at(0));
+}
+
+template<int req>
+struct triu1Dforward {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* out, const DType* data,
+                                  mshadow::Shape<2> oshape, int k) {
+    using namespace mxnet_op;
+
+    const index_t row_id = i / oshape[1];
+    const index_t col_id = i % oshape[1];
+    if (col_id < (row_id + k)) {
+      KERNEL_ASSIGN(out[i], req, static_cast<DType>(0));
+    } else {
+      KERNEL_ASSIGN(out[i], req, data[col_id]);
+    }
+  }
+};
+
+template<int req>
+struct triu1Dbackward {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* out, const DType* data,
+                                  mshadow::Shape<1> oshape, int k) {
+    using namespace mxnet_op;
+    auto m = oshape[0];
+    auto start = i - k;
+    DType res = 0;
+    for (auto y = 0; y <= start && y < m; y++) {
+      res += data[y * m + i];
+    }
+    KERNEL_ASSIGN(out[i], req, res);
+  }
+};
+
+template<int req>
+struct triu2D {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* out, const DType* data,
+                                  mshadow::Shape<2> oshape, int k) {
+    using namespace mxnet_op;
+
+    const index_t row_id = i / oshape[1];
+    const index_t col_id = i % oshape[1];
+    if (col_id < (row_id + k)) {
+      KERNEL_ASSIGN(out[i], req, static_cast<DType>(0));
+    } else {
+      KERNEL_ASSIGN(out[i], req, data[i]);
+    }
+  }
+};
+
+template<int req>
+struct triu3D {
+  template<typename DType>
+  MSHADOW_XINLINE static void Map(index_t i, DType* out, const DType* data,
+                                  mshadow::Shape<3> oshape, int k) {
+    using namespace mxnet_op;
+
+    const index_t row_id = i % (oshape[1] * oshape[2]) / oshape[2];
+    const index_t col_id = i % (oshape[1] * oshape[2]) % oshape[2];
+    if (col_id < (row_id + k)) {
+      KERNEL_ASSIGN(out[i], req, static_cast<DType>(0));
+    } else {
+      KERNEL_ASSIGN(out[i], req, data[i]);
+    }
+  }
+};
+
+template<typename xpu, bool back>
+void TriuOpProcess(const TBlob& in_data,
+                   const TBlob& out_data,
+                   index_t dsize,
+                   const TriuParam& param,
+                   mxnet_op::Stream<xpu> *s,
+                   const std::vector<OpReqType>& req) {
+  using namespace mxnet_op;
+  using namespace mshadow;
+
+  const mxnet::TShape& ishape = in_data.shape_;
+  const mxnet::TShape& oshape = out_data.shape_;
+
+  if (ishape.ndim() == 2 && oshape.ndim() == 2) {
+    MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        Kernel<triu2D<req_type>, xpu>::Launch(
+            s, dsize, out_data.dptr<DType>(), in_data.dptr<DType>(),
+            Shape2(oshape[0], oshape[1]), param.k);
+      });
+    });
+  } else if (ishape.ndim() > 2) {
+    MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        Kernel<triu3D<req_type>, xpu>::Launch(
+            s, dsize, out_data.dptr<DType>(), in_data.dptr<DType>(),
+            oshape.FlatTo3D(oshape.ndim() - 2), param.k);
+      });
+    });
+  } else {
+    MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, {
+      MXNET_ASSIGN_REQ_SWITCH(req[0], req_type, {
+        if (back) {
+          Kernel<triu1Dbackward<req_type>, xpu>::Launch(
+              s, dsize, out_data.dptr<DType>(), in_data.dptr<DType>(),
+              Shape1(oshape[0]), param.k);
+        } else {
+          Kernel<triu1Dforward<req_type>, xpu>::Launch(
+              s, dsize, out_data.dptr<DType>(), in_data.dptr<DType>(),
+              Shape2(oshape[0], oshape[1]), param.k);
+        }
+      });
+    });
+  }
+}
+
+template<typename xpu>
+void TriuOpForward(const nnvm::NodeAttrs& attrs,
+                   const OpContext& ctx,
+                   const std::vector<TBlob>& inputs,
+                   const std::vector<OpReqType>& req,
+                   const std::vector<TBlob>& outputs) {
+  using namespace mxnet_op;
+  using namespace mshadow;
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  CHECK_EQ(req.size(), 1U);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+  const TBlob& in_data = inputs[0];
+  const TBlob& out_data = outputs[0];
+  const TriuParam& param = nnvm::get<TriuParam>(attrs.parsed);
+
+  TriuOpProcess<xpu, false>(in_data, out_data, out_data.Size(), param, s, req);
+}
+
+template<typename xpu>
+void TriuOpBackward(const nnvm::NodeAttrs& attrs,
+                    const OpContext& ctx,
+                    const std::vector<TBlob>& inputs,
+                    const std::vector<OpReqType>& req,
+                    const std::vector<TBlob>& outputs) {
+  using namespace mxnet_op;
+  using namespace mshadow;
+  CHECK_EQ(inputs.size(), 1U);
+  CHECK_EQ(outputs.size(), 1U);
+  Stream<xpu> *s = ctx.get_stream<xpu>();
+
+  const TBlob& in_data = inputs[0];
+  const TBlob& out_data = outputs[0];
+  const TriuParam& param = nnvm::get<TriuParam>(attrs.parsed);
+
+  TriuOpProcess<xpu, true>(in_data, out_data, out_data.Size(), param, s, req);
+}
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_OPERATOR_NUMPY_NP_TRIU_OP_INL_H_
diff --git a/src/operator/numpy/np_triu_op.cc b/src/operator/numpy/np_triu_op.cc
new file mode 100644
index 000000000000..fdd526060001
--- /dev/null
+++ b/src/operator/numpy/np_triu_op.cc
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+* Copyright (c) 2020 by Contributors
+* \file np_triu_op.cc
+* \brief CPU implementation of numpy triu operator
+*/
+
+#include "./np_triu_op-inl.h"
+
+namespace mxnet {
+namespace op {
+
+DMLC_REGISTER_PARAMETER(TriuParam);
+
+NNVM_REGISTER_OP(_npi_triu)
+.set_attr_parser(ParamParser<TriuParam>)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr<nnvm::FListInputNames>("FListInputNames",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::string>{"data"};
+  })
+.set_attr<mxnet::FInferShape>("FInferShape", TriuOpShape)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<FCompute>("FCompute<cpu>", TriuOpForward<cpu>)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs) {
+    return std::vector<std::pair<int, int> >{{0, 0}};
+  })
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_triu"})
+.add_argument("data", "NDArray-or-Symbol", "Input ndarray")
+.add_arguments(TriuParam::__FIELDS__());
+
+
+NNVM_REGISTER_OP(_backward_triu)
+.set_attr_parser(ParamParser<TriuParam>)
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<FCompute>("FCompute<cpu>", TriuOpBackward<cpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/src/operator/numpy/np_triu_op.cu b/src/operator/numpy/np_triu_op.cu
new file mode 100644
index 000000000000..a143859e1db6
--- /dev/null
+++ b/src/operator/numpy/np_triu_op.cu
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file np_triu_op.cu
+ * \brief GPU implementation of numpy triu operator
+ */
+
+#include "./np_triu_op-inl.h"
+
+namespace mxnet {
+namespace op {
+
+NNVM_REGISTER_OP(_npi_triu)
+.set_attr<FCompute>("FCompute<gpu>", TriuOpForward<gpu>);
+
+NNVM_REGISTER_OP(_backward_triu)
+.set_attr<FCompute>("FCompute<gpu>", TriuOpBackward<gpu>);
+
+}  // namespace op
+}  // namespace mxnet
diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py
index 79a45f5e46f6..080fb03a7158 100644
--- a/tests/python/unittest/test_numpy_interoperability.py
+++ b/tests/python/unittest/test_numpy_interoperability.py
@@ -722,6 +722,23 @@ def _add_workload_tril():
         OpArgMngr.add_workload('tril', np.zeros((3, 3), dtype=dt))
 
 
+def _add_workload_triu():
+    OpArgMngr.add_workload('triu', np.random.uniform(size=(4, 1)))
+    for dt in ['float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8']:
+        OpArgMngr.add_workload('triu', np.ones((2, 2), dtype=dt))
+        a = np.array([
+            [[1, 1], [1, 1]],
+            [[1, 1], [1, 0]],
+            [[1, 1], [0, 0]],
+        ], dtype=dt)
+        OpArgMngr.add_workload('triu', a)
+        arr = np.array([[1, 1, np.inf],
+                        [1, 1, 1],
+                        [np.inf, 1, 1]])
+        OpArgMngr.add_workload('triu', arr)
+        OpArgMngr.add_workload('triu', np.zeros((3, 3), dtype=dt))
+
+
 def _add_workload_einsum():
     chars = 'abcdefghij'
     sizes = [2, 3, 4, 5, 4, 3, 2, 6, 5, 4]
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index ba6f1baef545..f57313c94276 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -2240,6 +2240,7 @@ def hybrid_forward(self, F, x):
                 ret_mx = np.tril(data_mx, k*prefix)
             assert same(ret_mx.asnumpy(), ret_np)
             ret_mx.backward()
+            print(data_mx.grad)
             if len(shape) == 2:
                 grad_np = _np.tri(*shape, k=k*prefix)
                 assert same(data_mx.grad.asnumpy(), grad_np)
@@ -2256,6 +2257,67 @@ def hybrid_forward(self, F, x):
                 assert same(ret_mx.asnumpy(), ret_np)
 
 
+@with_seed()
+@use_np
+def test_np_triu():
+    # numpy triu does not support scalar array (zero-dim)
+    config = [
+        ((4, 2), 3),
+        ((4, 2), 9),
+        ((4, 2), 0),
+        ((4, 2), -1),
+        ((4, 5, 6), 0),
+        ((4, 5, 6), 5),
+        ((4, 5, 6), 2),
+        ((4, 5, 6), -2),
+        ((4, 5, 6), -5),
+        ((4, 0), 0),
+        ((4, 0), 2),
+        ((4, 0), 4),
+        ((4, 0), -3),
+        ((4, 0, 5), 0),
+        ((4, 0, 5), 1),
+        ((4, 0, 5), 5),
+        ((4, 0, 5), -3),
+        ((3, ), 0),
+        ((3, ), 2),
+        ((3, ), 5)
+    ]
+
+    class TestTriu(HybridBlock):
+        def __init__(self, k):
+            super(TestTriu, self).__init__()
+            self._k = k
+
+        def hybrid_forward(self, F, x):
+            return F.np.triu(x, k=self._k)
+
+    for prefix in [1, -1]:
+        for shape, k in config:
+            data_np = _np.random.uniform(size=shape)
+            data_mx = np.array(data_np, dtype=data_np.dtype)
+            data_mx.attach_grad()
+            ret_np = _np.triu(data_np, k*prefix)
+            with mx.autograd.record():
+                ret_mx = np.triu(data_mx, k*prefix)
+            assert same(ret_mx.asnumpy(), ret_np)
+            ret_mx.backward()
+            if len(shape) == 2:
+                grad_np = _np.triu(_np.ones_like(data_np), k*prefix)
+                assert same(data_mx.grad.asnumpy(), grad_np)
+            if len(shape) == 1:
+                grad_np = _np.triu(_np.ones(shape), k*prefix)
+                grad_np = grad_np.sum(axis=0, keepdims=False)
+                assert same(data_mx.grad.asnumpy(), grad_np)
+
+            net = TestTriu(k*prefix)
+            for hybrid in [False, True]:
+                if hybrid:
+                    net.hybridize()
+                ret_mx = net(data_mx)
+                assert same(ret_mx.asnumpy(), ret_np)
+
+
 @with_seed()
 @use_np
 def test_np_unary_funcs():

From 9ec1c4b7ccdb908be6fe39c17c7fd17ee9f2d708 Mon Sep 17 00:00:00 2001
From: Yijun Chen <BenjaminCHEN2016@outlook.com>
Date: Sun, 10 May 2020 03:20:47 +0800
Subject: [PATCH 40/44] fix mixed type backward (#18250)

---
 src/operator/mshadow_op.h                     | 52 +++++++++++++++
 .../numpy/np_elemwise_broadcast_op.cc         | 64 +++++++++++++++++--
 .../numpy/np_elemwise_broadcast_op.cu         | 23 ++++++-
 src/operator/numpy/np_true_divide-inl.h       |  1 +
 src/operator/numpy/np_true_divide.cc          | 18 +++++-
 src/operator/numpy/np_true_divide.cu          |  4 ++
 src/operator/operator_tune.cc                 |  2 +
 tests/python/unittest/test_numpy_op.py        |  9 ++-
 8 files changed, 163 insertions(+), 10 deletions(-)

diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index 4d9de29ce709..59707f89dd19 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -728,6 +728,10 @@ MXNET_BINARY_MATH_OP_NC(minus_sign, a - b > DType(0) ? DType(1) : -DType(1));
 
 MXNET_BINARY_MATH_OP(rminus, b - a);
 
+MXNET_BINARY_MATH_OP_NC(posone, 1);
+
+MXNET_BINARY_MATH_OP_NC(negone, -1);
+
 MXNET_BINARY_MATH_OP(div_grad, 1.0f / math::id(b));
 
 template<>
@@ -795,6 +799,54 @@ struct mod : public mxnet_op::tunable {
   }
 };
 
+#ifndef _WIN32
+struct mixed_mod {
+  template<typename DType,
+           typename std::enable_if<std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static mshadow::half::half_t Map(DType a, mshadow::half::half_t b) {
+    return mod::Map(static_cast<mshadow::half::half_t>(a), b);
+  }
+
+  template<typename DType,
+           typename std::enable_if<std::is_same<DType, mshadow::half::half_t>::value ||
+                                   std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static float Map(DType a, float b) {
+    return mod::Map(static_cast<float>(a), b);
+  }
+
+  template<typename DType,
+           typename std::enable_if<std::is_same<DType, mshadow::half::half_t>::value ||
+                                   std::is_same<DType, float>::value ||
+                                   std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static double Map(DType a, double b) {
+    return mod::Map(static_cast<double>(a), b);
+  }
+};
+
+struct mixed_rmod {
+  template<typename DType,
+           typename std::enable_if<std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static mshadow::half::half_t Map(DType a, mshadow::half::half_t b) {
+    return mod::Map(b, static_cast<mshadow::half::half_t>(a));
+  }
+
+  template<typename DType,
+           typename std::enable_if<std::is_same<DType, mshadow::half::half_t>::value ||
+                                   std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static float Map(DType a, float b) {
+    return mod::Map(b, static_cast<float>(a));
+  }
+
+  template<typename DType,
+           typename std::enable_if<std::is_same<DType, mshadow::half::half_t>::value ||
+                                   std::is_same<DType, float>::value ||
+                                   std::is_integral<DType>::value, int>::type = 0>
+  MSHADOW_XINLINE static double Map(DType a, double b) {
+    return mod::Map(b, static_cast<double>(a));
+  }
+};
+#endif
+
 struct fmod : public mxnet_op::tunable {
   template<typename DType>
   MSHADOW_XINLINE static DType Map(DType a, DType b) {
diff --git a/src/operator/numpy/np_elemwise_broadcast_op.cc b/src/operator/numpy/np_elemwise_broadcast_op.cc
index 0ee677adbc13..bdf25a8508dd 100644
--- a/src/operator/numpy/np_elemwise_broadcast_op.cc
+++ b/src/operator/numpy/np_elemwise_broadcast_op.cc
@@ -116,7 +116,22 @@ MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_add)
   "FCompute<cpu>",
   NumpyBinaryBroadcastComputeWithBool<cpu, op::mshadow_op::plus>)
 #endif
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_broadcast_add"});
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_broadcast_add"});
+
+NNVM_REGISTER_OP(_backward_npi_broadcast_add)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 0}, {0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyBinaryBackwardUseIn<cpu, mshadow_op::posone,
+                                                                mshadow_op::posone>);
 
 MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_subtract)
 #ifndef _WIN32
@@ -129,7 +144,22 @@ MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_subtract)
   "FCompute<cpu>",
   NumpyBinaryBroadcastCompute<cpu, op::mshadow_op::minus>)
 #endif
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_broadcast_sub"});
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_broadcast_sub"});
+
+NNVM_REGISTER_OP(_backward_npi_broadcast_sub)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 0}, {0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyBinaryBackwardUseIn<cpu, mshadow_op::posone,
+                                                                mshadow_op::negone>);
 
 MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_multiply)
 #ifndef _WIN32
@@ -159,9 +189,33 @@ NNVM_REGISTER_OP(_backward_npi_broadcast_mul)
 .set_attr<FCompute>("FCompute<cpu>", NumpyBinaryBackwardUseIn<cpu, mshadow_op::right,
                                                               mshadow_op::left>);
 
-MXNET_OPERATOR_REGISTER_BINARY_BROADCAST(_npi_mod)
-.set_attr<FCompute>("FCompute<cpu>", BinaryBroadcastCompute<cpu, mshadow_op::mod>)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_broadcast_mod"});
+MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_mod)
+#ifndef _WIN32
+.set_attr<FCompute>(
+  "FCompute<cpu>",
+  NumpyBinaryBroadcastCompute<cpu, op::mshadow_op::mod, op::mshadow_op::mixed_mod,
+                                      op::mshadow_op::mixed_rmod>)
+#else
+.set_attr<FCompute>(
+  "FCompute<cpu>",
+  NumpyBinaryBroadcastCompute<cpu, op::mshadow_op::mod>)
+#endif
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_broadcast_mod"});
+
+NNVM_REGISTER_OP(_backward_npi_broadcast_mod)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyBinaryBackwardUseIn<cpu, mshadow_op::mod_grad,
+                                                              mshadow_op::mod_rgrad>);
 
 MXNET_OPERATOR_REGISTER_NP_BINARY_MIXED_PRECISION(_npi_power)
 #ifndef _WIN32
diff --git a/src/operator/numpy/np_elemwise_broadcast_op.cu b/src/operator/numpy/np_elemwise_broadcast_op.cu
index 1e0130494469..8a13b42e4846 100644
--- a/src/operator/numpy/np_elemwise_broadcast_op.cu
+++ b/src/operator/numpy/np_elemwise_broadcast_op.cu
@@ -40,6 +40,10 @@ NNVM_REGISTER_OP(_npi_add)
   NumpyBinaryBroadcastComputeWithBool<gpu, op::mshadow_op::plus>);
 #endif
 
+NNVM_REGISTER_OP(_backward_npi_broadcast_add)
+.set_attr<FCompute>("FCompute<gpu>", NumpyBinaryBackwardUseIn<gpu, mshadow_op::posone,
+                                                                mshadow_op::posone>);
+
 NNVM_REGISTER_OP(_npi_subtract)
 #ifndef _WIN32
 .set_attr<FCompute>(
@@ -52,6 +56,10 @@ NNVM_REGISTER_OP(_npi_subtract)
   NumpyBinaryBroadcastCompute<gpu, op::mshadow_op::minus>);
 #endif
 
+NNVM_REGISTER_OP(_backward_npi_broadcast_sub)
+.set_attr<FCompute>("FCompute<gpu>", NumpyBinaryBackwardUseIn<gpu, mshadow_op::posone,
+                                                                mshadow_op::negone>);
+
 NNVM_REGISTER_OP(_npi_multiply)
 #ifndef _WIN32
 .set_attr<FCompute>(
@@ -69,7 +77,20 @@ NNVM_REGISTER_OP(_backward_npi_broadcast_mul)
                                                               mshadow_op::left>);
 
 NNVM_REGISTER_OP(_npi_mod)
-.set_attr<FCompute>("FCompute<gpu>", BinaryBroadcastCompute<gpu, mshadow_op::mod>);
+#ifndef _WIN32
+.set_attr<FCompute>(
+  "FCompute<gpu>",
+  NumpyBinaryBroadcastCompute<gpu, op::mshadow_op::mod, op::mshadow_op::mixed_mod,
+                                      op::mshadow_op::mixed_rmod>);
+#else
+.set_attr<FCompute>(
+  "FCompute<gpu>",
+  NumpyBinaryBroadcastCompute<gpu, op::mshadow_op::mod>);
+#endif
+
+NNVM_REGISTER_OP(_backward_npi_broadcast_mod)
+.set_attr<FCompute>("FCompute<gpu>", NumpyBinaryBackwardUseIn<gpu, mshadow_op::mod_grad,
+                                                              mshadow_op::mod_rgrad>);
 
 NNVM_REGISTER_OP(_npi_power)
 #ifndef _WIN32
diff --git a/src/operator/numpy/np_true_divide-inl.h b/src/operator/numpy/np_true_divide-inl.h
index be2ce51506a1..538a026b6b8e 100644
--- a/src/operator/numpy/np_true_divide-inl.h
+++ b/src/operator/numpy/np_true_divide-inl.h
@@ -29,6 +29,7 @@
 #include <vector>
 #include "../../common/utils.h"
 #include "../tensor/elemwise_binary_broadcast_op.h"
+#include "../numpy/np_elemwise_broadcast_op.h"
 
 namespace mxnet {
 namespace op {
diff --git a/src/operator/numpy/np_true_divide.cc b/src/operator/numpy/np_true_divide.cc
index 6edfb4dd0901..f2529b348a2c 100644
--- a/src/operator/numpy/np_true_divide.cc
+++ b/src/operator/numpy/np_true_divide.cc
@@ -81,10 +81,26 @@ NNVM_REGISTER_OP(_npi_true_divide)
   })
 #endif
 .set_attr<FCompute>("FCompute<cpu>", TrueDivideBroadcastCompute<cpu>)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_broadcast_div"})
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseIn{"_backward_npi_broadcast_div"})
 .add_argument("lhs", "NDArray-or-Symbol", "Dividend array")
 .add_argument("rhs", "NDArray-or-Symbol", "Divisor array");
 
+
+NNVM_REGISTER_OP(_backward_npi_broadcast_div)
+.set_num_inputs(3)
+.set_num_outputs(2)
+.set_attr<nnvm::TIsBackward>("TIsBackward", true)
+.set_attr<nnvm::FInplaceOption>("FInplaceOption",
+  [](const NodeAttrs& attrs){
+    return std::vector<std::pair<int, int> >{{0, 1}};
+  })
+.set_attr<FResourceRequest>("FResourceRequest",
+  [](const NodeAttrs& attrs) {
+    return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
+  })
+.set_attr<FCompute>("FCompute<cpu>", NumpyBinaryBackwardUseIn<cpu, mshadow_op::div_grad,
+                                                              mshadow_op::div_rgrad>);
+
 NNVM_REGISTER_OP(_npi_true_divide_scalar)
 .set_num_inputs(1)
 .set_num_outputs(1)
diff --git a/src/operator/numpy/np_true_divide.cu b/src/operator/numpy/np_true_divide.cu
index 7211f4a0a006..c8eccfe140b4 100644
--- a/src/operator/numpy/np_true_divide.cu
+++ b/src/operator/numpy/np_true_divide.cu
@@ -31,6 +31,10 @@ namespace op {
 NNVM_REGISTER_OP(_npi_true_divide)
 .set_attr<FCompute>("FCompute<gpu>", TrueDivideBroadcastCompute<gpu>);
 
+NNVM_REGISTER_OP(_backward_npi_broadcast_div)
+.set_attr<FCompute>("FCompute<gpu>", NumpyBinaryBackwardUseIn<gpu, mshadow_op::div_grad,
+                                                              mshadow_op::div_rgrad>);
+
 NNVM_REGISTER_OP(_npi_true_divide_scalar)
 .set_attr<FCompute>("FCompute<gpu>", TrueDivideScalarCompute<gpu, mshadow_op::true_divide>);
 
diff --git a/src/operator/operator_tune.cc b/src/operator/operator_tune.cc
index b76e341b9fc6..20bb4bb98322 100644
--- a/src/operator/operator_tune.cc
+++ b/src/operator/operator_tune.cc
@@ -425,6 +425,8 @@ IMPLEMENT_BINARY_WORKLOAD_FWD(mxnet::op::mshadow_op::rldexp);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::ldexp_grad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::ldexp_rgrad);  // NOLINT()
 IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::rldexp_grad);  // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::posone); // NOLINT()
+IMPLEMENT_BINARY_WORKLOAD_BWD(mxnet::op::mshadow_op::negone); // NOLINT()
 /*!
  * \brief Tuner objects, *not* automatically generated
  */
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index f57313c94276..88917aac0aca 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -2668,10 +2668,13 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
                             use_broadcast=False, equal_nan=True)
 
     funcs = {
-        'add': (-1.0, 1.0, None, None),
-        'subtract': (-1.0, 1.0, None, None),
+        'add': (-1.0, 1.0, lambda y, x1, x2: _np.ones(y.shape),
+                           lambda y, x1, x2: _np.ones(y.shape)),
+        'subtract': (-1.0, 1.0, lambda y, x1, x2: _np.ones(y.shape),
+                                lambda y, x1, x2: _np.ones(y.shape) * -1),
         'multiply': (-1.0, 1.0, lambda y, x1, x2: _np.broadcast_to(x2, y.shape),
                                 lambda y, x1, x2: _np.broadcast_to(x1, y.shape)),
+        'mod': (1.0, 5.0, None, None),
         'power': (1.0, 3.0, lambda y, x1, x2: _np.power(x1, x2 - 1.0) * x2,
                             lambda y, x1, x2: _np.power(x1, x2) * _np.log(x1)),
     }
@@ -2699,7 +2702,7 @@ def hybrid_forward(self, F, a, b, *args, **kwargs):
                     continue
                 check_mixed_precision_binary_func(func, low, high, lshape, rshape, lgrad, rgrad, type1, type2)
 
-            if func == 'subtract':
+            if func == 'subtract' or func == 'mod':
                 continue
             for type1, type2 in itertools.product(itypes, itypes):
                 if type1 == type2:

From 05551d0454eb6e89a444df9229edf84e70553ef2 Mon Sep 17 00:00:00 2001
From: Hao Jin <hjjn.amzn@gmail.com>
Date: Thu, 16 Apr 2020 12:32:44 -0700
Subject: [PATCH 41/44] [Numpy] Add ffi for np.sum, np.std, np.var, np.average
 and np.histogram (#17866)

* add ffi for sum, var and std

* add ffi wrapper for np.average

* add ffi wrapper for np.histogram
---
 benchmark/python/ffi/benchmark_ffi.py         |   5 +
 include/mxnet/runtime/ffi_helper.h            |  18 ++
 include/mxnet/runtime/object.h                |   1 +
 python/mxnet/_ffi/_cython/convert.pxi         |   6 +
 python/mxnet/_ffi/node_generic.py             |   2 +
 python/mxnet/_numpy_op_doc.py                 |  92 --------
 python/mxnet/ndarray/numpy/_op.py             | 114 +++++++++-
 python/mxnet/numpy/multiarray.py              | 104 ++++++++-
 python/mxnet/symbol/numpy/_symbol.py          |  51 ++++-
 python/mxnet/symbol/numpy/linalg.py           |   8 +-
 src/api/_api_internal/_api_internal.cc        |  10 +
 src/api/operator/numpy/np_bincount_op.cc      |   4 +-
 .../numpy/np_broadcast_reduce_op_value.cc     |  67 +++++-
 src/api/operator/numpy/np_cumsum.cc           |   4 +-
 src/api/operator/numpy/np_histogram_op.cc     |  81 +++++++
 src/api/operator/numpy/np_moments_op.cc       | 209 ++++++++++++++++++
 src/api/operator/numpy/np_tensordot_op.cc     |   4 +-
 src/api/operator/utils.h                      |  10 +
 src/operator/numpy/np_broadcast_reduce_op.h   |  32 ++-
 .../numpy/np_broadcast_reduce_op_value.cc     |  22 +-
 .../numpy/np_broadcast_reduce_op_value.cu     |   4 +-
 src/operator/tensor/histogram-inl.h           |  42 ++--
 22 files changed, 739 insertions(+), 151 deletions(-)
 create mode 100644 src/api/operator/numpy/np_histogram_op.cc
 create mode 100644 src/api/operator/numpy/np_moments_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index bc55164d9663..1e911598bbb1 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -60,6 +60,11 @@ def prepare_workloads():
     OpArgMngr.add_workload("tensordot", pool['2x2'], pool['2x2'], ((1, 0), (0, 1)))
     OpArgMngr.add_workload("kron", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("cumsum", pool['3x2'], axis=0, out=pool['3x2'])
+    OpArgMngr.add_workload("sum", pool['2x2'], axis=0, keepdims=True, out=pool['1x2'])
+    OpArgMngr.add_workload("std", pool['2x2'], axis=0, ddof=0, keepdims=True, out=pool['1x2'])
+    OpArgMngr.add_workload("var", pool['2x2'], axis=0, ddof=1, keepdims=True, out=pool['1x2'])
+    OpArgMngr.add_workload("average", pool['2x2'], weights=pool['2'], axis=1, returned=True)
+    OpArgMngr.add_workload("histogram", pool['2x2'], bins=10, range=(0.0, 10.0))
     OpArgMngr.add_workload("add", pool['2x2'], pool['2x2'])
     OpArgMngr.add_workload("linalg.eig", pool['3x3'])
     OpArgMngr.add_workload("linalg.eigh", pool['3x3'])
diff --git a/include/mxnet/runtime/ffi_helper.h b/include/mxnet/runtime/ffi_helper.h
index 49134ca122a7..cfc79a6c4f47 100644
--- a/include/mxnet/runtime/ffi_helper.h
+++ b/include/mxnet/runtime/ffi_helper.h
@@ -99,6 +99,24 @@ class Integer: public ObjectRef {
   MXNET_DEFINE_OBJECT_REF_METHODS(Integer, ObjectRef, IntegerObj)
 };
 
+class FloatObj: public Object {
+ public:
+  double value;
+  static constexpr const uint32_t _type_index = TypeIndex::kFloat;
+  static constexpr const char* _type_key = "MXNet.Float";
+  MXNET_DECLARE_FINAL_OBJECT_INFO(FloatObj, Object)
+};
+
+class Float: public ObjectRef {
+ public:
+  explicit Float(double value,
+                 ObjectPtr<FloatObj>&& data = make_object<FloatObj>()) {
+    data->value = value;
+    data_ = std::move(data);
+  }
+  MXNET_DEFINE_OBJECT_REF_METHODS(Float, ObjectRef, FloatObj)
+};
+
 //  Helper functions for fast FFI implementations
 /*!
  * \brief A builder class that helps to incrementally build ADT.
diff --git a/include/mxnet/runtime/object.h b/include/mxnet/runtime/object.h
index a031a56d88ed..48c9badb3ba7 100644
--- a/include/mxnet/runtime/object.h
+++ b/include/mxnet/runtime/object.h
@@ -58,6 +58,7 @@ enum TypeIndex  {
   kEllipsis = 5,
   kSlice = 6,
   kInteger = 7,
+  kFloat = 8,
   kStaticIndexEnd,
   /*! \brief Type index is allocated during runtime. */
   kDynamic = kStaticIndexEnd
diff --git a/python/mxnet/_ffi/_cython/convert.pxi b/python/mxnet/_ffi/_cython/convert.pxi
index 2cbdc48b49a8..d7b1ea5659dc 100644
--- a/python/mxnet/_ffi/_cython/convert.pxi
+++ b/python/mxnet/_ffi/_cython/convert.pxi
@@ -43,6 +43,10 @@ cdef extern from "mxnet/runtime/ffi_helper.h" namespace "mxnet::runtime":
         Integer()
         Integer(int64_t)
 
+    cdef cppclass Float(ObjectRef):
+        Float()
+        Float(double)
+
 
 cdef inline ADT convert_tuple(tuple src_tuple) except *:
     cdef uint32_t size = len(src_tuple)
@@ -71,5 +75,7 @@ cdef inline ObjectRef convert_object(object src_obj) except *:
         return convert_list(src_obj)
     elif isinstance(src_obj, Integral):
         return Integer(<int64_t>src_obj)
+    elif isinstance(src_obj, float):
+        return Float(<double>src_obj)
     else:
         raise TypeError("Don't know how to convert type %s" % type(src_obj))
diff --git a/python/mxnet/_ffi/node_generic.py b/python/mxnet/_ffi/node_generic.py
index c7f332390ce7..07b4825654d1 100644
--- a/python/mxnet/_ffi/node_generic.py
+++ b/python/mxnet/_ffi/node_generic.py
@@ -52,6 +52,8 @@ def convert_to_node(value):
     """
     if isinstance(value, Integral):
         return _api_internal._Integer(value)
+    elif isinstance(value, float):
+        return _api_internal._Float(value)
     elif isinstance(value, (list, tuple)):
         value = [convert_to_node(x) for x in value]
         return _api_internal._ADT(*value)
diff --git a/python/mxnet/_numpy_op_doc.py b/python/mxnet/_numpy_op_doc.py
index 8341d43608ce..857b87a7586f 100644
--- a/python/mxnet/_numpy_op_doc.py
+++ b/python/mxnet/_numpy_op_doc.py
@@ -231,98 +231,6 @@ def _np_dot(a, b, out=None):
     pass
 
 
-def _np_sum(a, axis=None, dtype=None, keepdims=False, initial=None, out=None):
-    r"""
-    Sum of array elements over a given axis.
-
-    Parameters
-    ----------
-    a : ndarray
-        Input data.
-    axis : None or int, optional
-        Axis or axes along which a sum is performed.  The default,
-        axis=None, will sum all of the elements of the input array.  If
-        axis is negative it counts from the last to the first axis.
-    dtype : dtype, optional
-        The type of the returned array and of the accumulator in which the
-        elements are summed. The default type is float32.
-    keepdims : bool, optional
-        If this is set to True, the axes which are reduced are left
-        in the result as dimensions with size one. With this option,
-        the result will broadcast correctly against the input array.
-
-        If the default value is passed, then `keepdims` will not be
-        passed through to the `sum` method of sub-classes of
-        `ndarray`, however any non-default value will be.  If the
-        sub-classes `sum` method does not implement `keepdims` any
-        exceptions will be raised.
-    initial: Currently only supports None as input, optional
-        Starting value for the sum.
-        Currently not implemented. Please use ``None`` as input or skip this argument.
-    out : ndarray or None, optional
-        Alternative output array in which to place the result. It must have
-        the same shape and dtype as the expected output.
-
-    Returns
-    -------
-    sum_along_axis : ndarray
-        An ndarray with the same shape as `a`, with the specified
-        axis removed. If an output array is specified, a reference to
-        `out` is returned.
-
-    Notes
-    -----
-    - Input type does not support Python native iterables.
-    - "out" param: cannot perform auto type change. out ndarray's dtype must be the same as the expected output.
-    - "initial" param is not supported yet. Please use None as input.
-    - Arithmetic is modular when using integer types, and no error is raised on overflow.
-    - The sum of an empty array is the neutral element 0:
-
-    >>> a = np.empty(1)
-    >>> np.sum(a)
-    array(0.)
-
-    This function differs from the original `numpy.sum
-    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.sum.html>`_ in
-    the following aspects:
-
-    - Input type does not support Python native iterables(list, tuple, ...).
-    - "out" param: cannot perform auto type cast. out ndarray's dtype must be the same as the expected output.
-    - "initial" param is not supported yet. Please use ``None`` as input or skip it.
-
-    Examples
-    --------
-    >>> a = np.array([0.5, 1.5])
-    >>> np.sum(a)
-    array(2.)
-    >>> a = np.array([0.5, 0.7, 0.2, 1.5])
-    >>> np.sum(a, dtype=np.int32)
-    array(2, dtype=int32)
-    >>> a = np.array([[0, 1], [0, 5]])
-    >>> np.sum(a)
-    array(6.)
-    >>> np.sum(a, axis=0)
-    array([0., 6.])
-    >>> np.sum(a, axis=1)
-    array([1., 5.])
-
-    With output ndarray:
-
-    >>> a = np.array([[0, 1], [0, 5]])
-    >>> b = np.ones((2,), dtype=np.float32)
-    >>> np.sum(a, axis = 0, out=b)
-    array([0., 6.])
-    >>> b
-    array([0., 6.])
-
-    If the accumulator is too small, overflow occurs:
-
-    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)
-    array(-128, dtype=int8)
-    """
-    pass
-
-
 def _np_copy(a, out=None):
     """
     Return an array copy of the given object.
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index c1ce9092904a..3074cf2b3502 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -48,7 +48,7 @@
            'true_divide', 'nonzero', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'interp',
            'diff', 'ediff1d', 'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'diag', 'diagonal']
+           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'sum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -1739,13 +1739,13 @@ def histogram(a, bins=10, range=None, normed=None, weights=None, density=None):
     if isinstance(bins, numeric_types):
         if range is None:
             raise NotImplementedError("automatic range is not supported yet...")
-        return _npi.histogram(a, bin_cnt=bins, range=range)
+        return tuple(_api_internal.histogram(a, None, bins, range))
     if isinstance(bins, (list, tuple)):
         raise NotImplementedError("array_like bins is not supported yet...")
     if isinstance(bins, str):
         raise NotImplementedError("string bins is not supported yet...")
     if isinstance(bins, NDArray):
-        return _npi.histogram(a, bins=bins)
+        return tuple(_api_internal.histogram(a, bins, None, None))
     raise ValueError("np.histogram fails with", locals())
 
 
@@ -4859,10 +4859,7 @@ def average(a, axis=None, weights=None, returned=False, out=None):
     >>> np.average(data, axis=1, weights=weights)
     array([0.75, 2.75, 4.75])
     """
-    if weights is None:
-        return _npi.average(a, axis=axis, weights=None, returned=returned, weighted=False, out=out)
-    else:
-        return _npi.average(a, axis=axis, weights=weights, returned=returned, out=out)
+    return _api_internal.average(a, weights, axis, returned, weights is not None, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -4987,7 +4984,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint:
     >>> np.std(a, dtype=np.float64)
     array(0.45, dtype=float64)
     """
-    return _npi.std(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
+    return _api_internal.std(a, axis, dtype, ddof, keepdims, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5057,7 +5054,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint:
     >>> ((1-0.55)**2 + (0.1-0.55)**2)/2
     0.2025
     """
-    return _npi.var(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
+    return _api_internal.var(a, axis, dtype, ddof, keepdims, out)
 
 
 # pylint: disable=redefined-outer-name
@@ -6257,7 +6254,7 @@ def outer(a, b):
         [-2., -1.,  0.,  1.,  2.],
         [-2., -1.,  0.,  1.,  2.]])
     """
-    return tensordot(a.flatten(), b.flatten(), 0)
+    return tensordot(a.reshape_view((-1, )), b.reshape_view((-1, )), 0)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -8427,3 +8424,100 @@ def diagonal(a, offset=0, axis1=0, axis2=1):
             [1, 7]])
     """
     return _api_internal.diagonal(a, offset, axis1, axis2)
+
+
+# pylint:disable=redefined-outer-name, too-many-arguments
+@set_module('mxnet.ndarray.numpy')
+def sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None, where=None):
+    r"""
+    Sum of array elements over a given axis.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input data.
+    axis : None or int, optional
+        Axis or axes along which a sum is performed.  The default,
+        axis=None, will sum all of the elements of the input array.  If
+        axis is negative it counts from the last to the first axis.
+    dtype : dtype, optional
+        The type of the returned array and of the accumulator in which the
+        elements are summed. The default type is float32.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+
+        If the default value is passed, then `keepdims` will not be
+        passed through to the `sum` method of sub-classes of
+        `ndarray`, however any non-default value will be.  If the
+        sub-classes `sum` method does not implement `keepdims` any
+        exceptions will be raised.
+    initial: Currently only supports None as input, optional
+        Starting value for the sum.
+        Currently not implemented. Please use ``None`` as input or skip this argument.
+    out : ndarray or None, optional
+        Alternative output array in which to place the result. It must have
+        the same shape and dtype as the expected output.
+
+    Returns
+    -------
+    sum_along_axis : ndarray
+        An ndarray with the same shape as `a`, with the specified
+        axis removed. If an output array is specified, a reference to
+        `out` is returned.
+
+    Notes
+    -----
+    - Input type does not support Python native iterables.
+    - "out" param: cannot perform auto type change. out ndarray's dtype must be the same as the expected output.
+    - "initial" param is not supported yet. Please use None as input.
+    - Arithmetic is modular when using integer types, and no error is raised on overflow.
+    - The sum of an empty array is the neutral element 0:
+
+    >>> a = np.empty(1)
+    >>> np.sum(a)
+    array(0.)
+
+    This function differs from the original `numpy.sum
+    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.sum.html>`_ in
+    the following aspects:
+
+    - Input type does not support Python native iterables(list, tuple, ...).
+    - "out" param: cannot perform auto type cast. out ndarray's dtype must be the same as the expected output.
+    - "initial" param is not supported yet. Please use ``None`` as input or skip it.
+
+    Examples
+    --------
+    >>> a = np.array([0.5, 1.5])
+    >>> np.sum(a)
+    array(2.)
+    >>> a = np.array([0.5, 0.7, 0.2, 1.5])
+    >>> np.sum(a, dtype=np.int32)
+    array(2, dtype=int32)
+    >>> a = np.array([[0, 1], [0, 5]])
+    >>> np.sum(a)
+    array(6.)
+    >>> np.sum(a, axis=0)
+    array([0., 6.])
+    >>> np.sum(a, axis=1)
+    array([1., 5.])
+
+    With output ndarray:
+
+    >>> a = np.array([[0, 1], [0, 5]])
+    >>> b = np.ones((2,), dtype=np.float32)
+    >>> np.sum(a, axis=0, out=b)
+    array([0., 6.])
+    >>> b
+    array([0., 6.])
+
+    If the accumulator is too small, overflow occurs:
+
+    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)
+    array(-128, dtype=int8)
+    """
+    if where is not None and where is not True:
+        raise ValueError("only where=None or where=True cases are supported for now")
+    return _api_internal.sum(a, axis, dtype, keepdims, initial, out)
+# pylint:enable=redefined-outer-name, too-many-arguments
diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index 57807f78e388..d8f7f4a29ce3 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -73,7 +73,7 @@
            'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d', 'resize', 'matmul',
            'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite', 'polyval', 'where', 'bincount',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'pad', 'cumsum', 'rollaxis', 'diag', 'diagonal']
+           'pad', 'cumsum', 'sum', 'rollaxis', 'diag', 'diagonal']
 
 __all__ += fallback.__all__
 
@@ -6843,7 +6843,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint:
     >>> np.std(a, dtype=np.float64)
     array(0.45, dtype=float64)
     """
-    return _npi.std(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
+    return _mx_nd_np.std(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
 # pylint: enable=redefined-outer-name
 
 
@@ -6964,7 +6964,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False):  # pylint:
     >>> ((1-0.55)**2 + (0.1-0.55)**2)/2
     0.2025
     """
-    return _npi.var(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
+    return _mx_nd_np.var(a, axis=axis, dtype=dtype, ddof=ddof, keepdims=keepdims, out=out)
 
 
 # pylint: disable=redefined-outer-name
@@ -7127,6 +7127,7 @@ def ravel(x, order='C'):
     return _mx_nd_np.ravel(x, order)
 
 
+@set_module('mxnet.numpy')
 def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-name
     """
     Converts a flat index or array of flat indices into a tuple of coordinate arrays.
@@ -7157,6 +7158,7 @@ def unravel_index(indices, shape, order='C'): # pylint: disable=redefined-outer-
     return _mx_nd_np.unravel_index(indices, shape, order=order)
 
 
+@set_module('mxnet.numpy')
 def flatnonzero(a):
     r"""
     Return indices that are non-zero in the flattened version of a.
@@ -7196,6 +7198,7 @@ def flatnonzero(a):
     return _mx_nd_np.flatnonzero(a)
 
 
+@set_module('mxnet.numpy')
 def diag_indices_from(arr):
     """
     This returns a tuple of indices that can be used to access the main diagonal of an array
@@ -10548,3 +10551,98 @@ def diagonal(a, offset=0, axis1=0, axis2=1):
             [1, 7]])
     """
     return _mx_nd_np.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)
+
+
+# pylint: disable=redefined-outer-name, too-many-arguments
+@set_module('mxnet.numpy')
+def sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None, where=None):
+    r"""
+    Sum of array elements over a given axis.
+
+    Parameters
+    ----------
+    a : ndarray
+        Input data.
+    axis : None or int, optional
+        Axis or axes along which a sum is performed.  The default,
+        axis=None, will sum all of the elements of the input array.  If
+        axis is negative it counts from the last to the first axis.
+    dtype : dtype, optional
+        The type of the returned array and of the accumulator in which the
+        elements are summed. The default type is float32.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+
+        If the default value is passed, then `keepdims` will not be
+        passed through to the `sum` method of sub-classes of
+        `ndarray`, however any non-default value will be.  If the
+        sub-classes `sum` method does not implement `keepdims` any
+        exceptions will be raised.
+    initial: Currently only supports None as input, optional
+        Starting value for the sum.
+        Currently not implemented. Please use ``None`` as input or skip this argument.
+    out : ndarray or None, optional
+        Alternative output array in which to place the result. It must have
+        the same shape and dtype as the expected output.
+
+    Returns
+    -------
+    sum_along_axis : ndarray
+        An ndarray with the same shape as `a`, with the specified
+        axis removed. If an output array is specified, a reference to
+        `out` is returned.
+
+    Notes
+    -----
+    - Input type does not support Python native iterables.
+    - "out" param: cannot perform auto type change. out ndarray's dtype must be the same as the expected output.
+    - "initial" param is not supported yet. Please use None as input.
+    - Arithmetic is modular when using integer types, and no error is raised on overflow.
+    - The sum of an empty array is the neutral element 0:
+
+    >>> a = np.empty(1)
+    >>> np.sum(a)
+    array(0.)
+
+    This function differs from the original `numpy.sum
+    <https://docs.scipy.org/doc/numpy/reference/generated/numpy.sum.html>`_ in
+    the following aspects:
+
+    - Input type does not support Python native iterables(list, tuple, ...).
+    - "out" param: cannot perform auto type cast. out ndarray's dtype must be the same as the expected output.
+    - "initial" param is not supported yet. Please use ``None`` as input or skip it.
+
+    Examples
+    --------
+    >>> a = np.array([0.5, 1.5])
+    >>> np.sum(a)
+    array(2.)
+    >>> a = np.array([0.5, 0.7, 0.2, 1.5])
+    >>> np.sum(a, dtype=np.int32)
+    array(2, dtype=int32)
+    >>> a = np.array([[0, 1], [0, 5]])
+    >>> np.sum(a)
+    array(6.)
+    >>> np.sum(a, axis=0)
+    array([0., 6.])
+    >>> np.sum(a, axis=1)
+    array([1., 5.])
+
+    With output ndarray:
+
+    >>> a = np.array([[0, 1], [0, 5]])
+    >>> b = np.ones((2,), dtype=np.float32)
+    >>> np.sum(a, axis = 0, out=b)
+    array([0., 6.])
+    >>> b
+    array([0., 6.])
+
+    If the accumulator is too small, overflow occurs:
+
+    >>> np.ones(128, dtype=np.int8).sum(dtype=np.int8)
+    array(-128, dtype=int8)
+    """
+    return _mx_nd_np.sum(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims, initial=initial, where=where)
+# pylint: enable=redefined-outer-name, too-many-arguments
diff --git a/python/mxnet/symbol/numpy/_symbol.py b/python/mxnet/symbol/numpy/_symbol.py
index aae0ed21efc1..bf1d31329c4e 100644
--- a/python/mxnet/symbol/numpy/_symbol.py
+++ b/python/mxnet/symbol/numpy/_symbol.py
@@ -54,7 +54,7 @@
            'true_divide', 'quantile', 'percentile', 'shares_memory', 'may_share_memory', 'diff', 'ediff1d',
            'resize', 'polyval', 'nan_to_num', 'isnan', 'isinf', 'isposinf', 'isneginf', 'isfinite',
            'atleast_1d', 'atleast_2d', 'atleast_3d',
-           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'diag', 'diagonal']
+           'where', 'bincount', 'rollaxis', 'pad', 'cumsum', 'sum', 'diag', 'diagonal']
 
 
 @set_module('mxnet.symbol.numpy')
@@ -650,7 +650,7 @@ def diag(self, k=0, **kwargs):
 
     def sum(self, axis=None, dtype=None, out=None, keepdims=False):  # pylint: disable=arguments-differ
         """Return the sum of the array elements over the given axis."""
-        return _mx_np_op.sum(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
+        return _npi.sum(self, axis=axis, dtype=dtype, out=out, keepdims=keepdims)
 
     def nansum(self, *args, **kwargs):
         """Convenience fluent method for :py:func:`nansum`.
@@ -7295,4 +7295,51 @@ def diagonal(a, offset=0, axis1=0, axis2=1):
     return _npi.diagonal(a, offset=offset, axis1=axis1, axis2=axis2)
 
 
+# pylint:disable=redefined-outer-name, too-many-arguments
+@set_module('mxnet.symbol.numpy')
+def sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None, where=None):
+    r"""
+    Sum of array elements over a given axis.
+
+    Parameters
+    ----------
+    a : _Symbol
+        Input data.
+    axis : None or int, optional
+        Axis or axes along which a sum is performed.  The default,
+        axis=None, will sum all of the elements of the input array.  If
+        axis is negative it counts from the last to the first axis.
+    dtype : dtype, optional
+        The type of the returned array and of the accumulator in which the
+        elements are summed. The default type is float32.
+    keepdims : bool, optional
+        If this is set to True, the axes which are reduced are left
+        in the result as dimensions with size one. With this option,
+        the result will broadcast correctly against the input array.
+
+        If the default value is passed, then `keepdims` will not be
+        passed through to the `sum` method of sub-classes of
+        `ndarray`, however any non-default value will be.  If the
+        sub-classes `sum` method does not implement `keepdims` any
+        exceptions will be raised.
+    initial: Currently only supports None as input, optional
+        Starting value for the sum.
+        Currently not implemented. Please use ``None`` as input or skip this argument.
+    out : ndarray or None, optional
+        Alternative output array in which to place the result. It must have
+        the same shape and dtype as the expected output.
+
+    Returns
+    -------
+    sum_along_axis : _Symbol
+        An ndarray with the same shape as `a`, with the specified
+        axis removed. If an output array is specified, a reference to
+        `out` is returned.
+    """
+    if where is not None and where is not True:
+        raise ValueError("only where=None or where=True cases are supported for now")
+    return _npi.sum(a, axis=axis, dtype=dtype, keepdims=keepdims, initial=initial, out=out)
+# pylint:enable=redefined-outer-name, too-many-arguments
+
+
 _set_np_symbol_class(_Symbol)
diff --git a/python/mxnet/symbol/numpy/linalg.py b/python/mxnet/symbol/numpy/linalg.py
index 3cea6ddae157..1fbac50b630a 100644
--- a/python/mxnet/symbol/numpy/linalg.py
+++ b/python/mxnet/symbol/numpy/linalg.py
@@ -324,18 +324,18 @@ def norm(x, ord=None, axis=None, keepdims=False):
                         if row_axis > col_axis:
                             row_axis -= 1
                     if ord == 'inf':
-                        return _mx_sym_np.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).max(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                        return _npi.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).max(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
                     else:
-                        return _mx_sym_np.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).min(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                        return _npi.sum(_symbol.abs(x), axis=col_axis, keepdims=keepdims).min(axis=row_axis, keepdims=keepdims)  # pylint: disable=line-too-long
                 if ord in [1, -1]:
                     row_axis, col_axis = axis
                     if not keepdims:
                         if row_axis < col_axis:
                             col_axis -= 1
                     if ord == 1:
-                        return _mx_sym_np.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).max(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                        return _npi.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).max(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
                     elif ord == -1:
-                        return _mx_sym_np.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).min(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
+                        return _npi.sum(_symbol.abs(x), axis=row_axis, keepdims=keepdims).min(axis=col_axis, keepdims=keepdims)  # pylint: disable=line-too-long
                 if ord in [2, -2]:
                     return _npi.norm(x, ord=ord, axis=axis, keepdims=keepdims, flag=0)
                 if ord is None:
diff --git a/src/api/_api_internal/_api_internal.cc b/src/api/_api_internal/_api_internal.cc
index 586dce82f383..7e1ce045f353 100644
--- a/src/api/_api_internal/_api_internal.cc
+++ b/src/api/_api_internal/_api_internal.cc
@@ -43,6 +43,16 @@ MXNET_REGISTER_GLOBAL("_Integer")
     }
 });
 
+MXNET_REGISTER_GLOBAL("_Float")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+    using namespace runtime;
+    if (args[0].type_code() == kDLFloat) {
+      *ret = Float(args[0].operator double());
+    } else {
+      LOG(FATAL) << "only accept float";
+    }
+});
+
 MXNET_REGISTER_GLOBAL("_ADT")
 .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
     using namespace runtime;
diff --git a/src/api/operator/numpy/np_bincount_op.cc b/src/api/operator/numpy/np_bincount_op.cc
index afa3278c24e4..7be884aefb1a 100644
--- a/src/api/operator/numpy/np_bincount_op.cc
+++ b/src/api/operator/numpy/np_bincount_op.cc
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
index c2d87a285cde..4cd2e485d987 100644
--- a/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
+++ b/src/api/operator/numpy/np_broadcast_reduce_op_value.cc
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,13 +18,14 @@
  */
 
 /*!
- * \file broadcast_reduce_op_value.cc
+ * \file np_broadcast_reduce_op_value.cc
  * \brief Implementation of the API of functions in
  * src/operator/tensor/np_broadcast_reduce_op_value.cc
  */
 #include <mxnet/api_registry.h>
 #include <mxnet/runtime/packed_func.h>
 #include "../utils.h"
+#include "../../../operator/tensor/broadcast_reduce_op.h"
 #include "../../../operator/numpy/np_broadcast_reduce_op.h"
 
 namespace mxnet {
@@ -51,6 +52,65 @@ MXNET_REGISTER_API("_npi.broadcast_to")
   *ret = ndoutputs[0];
 });
 
+MXNET_REGISTER_API("_npi.sum")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_sum");
+  op::NumpyReduceAxesParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+
+  // parse axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    if (args[1].type_code() == kDLInt) {
+      param.axis = Tuple<int>(1, args[1].operator int64_t());
+    } else {
+      param.axis = Tuple<int>(args[1].operator ObjectRef());
+    }
+  }
+
+  // parse dtype
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::nullopt;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+
+  // parse keepdims
+  if (args[3].type_code() == kNull) {
+    param.keepdims = false;
+  } else {
+    param.keepdims = args[3].operator bool();
+  }
+
+  // parse initial
+  if (args[4].type_code() == kNull) {
+    param.initial = dmlc::nullopt;
+  } else {
+    param.initial = args[4].operator double();
+  }
+
+  attrs.parsed = std::move(param);
+
+  SetAttrDict<op::NumpyReduceAxesParam>(&attrs);
+
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+  int num_inputs = 1;
+
+  NDArray* outputs[] = {args[5].operator NDArray*()};
+  NDArray** out = (outputs[0] == nullptr) ? nullptr : outputs;
+  int num_outputs = (outputs[0] != nullptr);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
 MXNET_REGISTER_API("_npi.mean")
 .set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
   using namespace runtime;
@@ -67,6 +127,7 @@ MXNET_REGISTER_API("_npi.mean")
   } else {
     param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
   }
+
   if (args[3].type_code() == kNull) {
     param.keepdims = false;
   } else {
diff --git a/src/api/operator/numpy/np_cumsum.cc b/src/api/operator/numpy/np_cumsum.cc
index 0ef3b3fdf7bf..d0b200c66fd4 100644
--- a/src/api/operator/numpy/np_cumsum.cc
+++ b/src/api/operator/numpy/np_cumsum.cc
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/src/api/operator/numpy/np_histogram_op.cc b/src/api/operator/numpy/np_histogram_op.cc
new file mode 100644
index 000000000000..b517cce80803
--- /dev/null
+++ b/src/api/operator/numpy/np_histogram_op.cc
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_histogram_op.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/histogram.cc
+ */
+
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/tensor/histogram-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.histogram")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  nnvm::NodeAttrs attrs;
+  const nnvm::Op* op = Op::Get("_npi_histogram");
+  op::HistogramParam param;
+  // parse bin_cnt
+  if (args[2].type_code() == kNull) {
+    param.bin_cnt = dmlc::nullopt;
+  } else {
+    param.bin_cnt = args[2].operator int();
+  }
+
+  // parse range
+  if (args[3].type_code() == kNull) {
+    param.range = dmlc::nullopt;
+  } else {
+    param.range = Obj2Tuple<double, Float>(args[3].operator ObjectRef());
+  }
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  SetAttrDict<op::HistogramParam>(&attrs);
+
+  std::vector<NDArray*> inputs_vec;
+  int num_inputs = 0;
+
+  if (args[2].type_code() != kNull) {
+    CHECK_EQ(args[1].type_code(), kNull)
+      << "bins should be None when bin_cnt is provided";
+    inputs_vec.push_back((args[0].operator NDArray*()));
+    num_inputs = 1;
+  } else {
+    CHECK_NE(args[1].type_code(), kNull)
+      << "bins should not be None when bin_cnt is not provided";
+    // inputs
+    inputs_vec.push_back((args[0].operator NDArray*()));
+    inputs_vec.push_back((args[1].operator NDArray*()));
+    num_inputs = 2;
+  }
+
+  // outputs
+  NDArray** out = nullptr;
+  int num_outputs = 0;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs_vec.data(), &num_outputs, out);
+  *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                 NDArrayHandle(ndoutputs[1])});
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/np_moments_op.cc b/src/api/operator/numpy/np_moments_op.cc
new file mode 100644
index 000000000000..e4e9238bb6c1
--- /dev/null
+++ b/src/api/operator/numpy/np_moments_op.cc
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_moments_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_moments_op.cc
+ */
+
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/numpy/np_broadcast_reduce_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.std")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_std");
+  op::NumpyMomentsParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+
+  // parse axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    if (args[1].type_code() == kDLInt) {
+      param.axis = Tuple<int>(1, args[1].operator int64_t());
+    } else {
+      param.axis = Tuple<int>(args[1].operator ObjectRef());
+    }
+  }
+
+  // parse dtype
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::nullopt;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+
+  // parse ddof
+  param.ddof = args[3].operator int();
+
+  // parse keepdims
+  if (args[4].type_code() == kNull) {
+    param.keepdims = false;
+  } else {
+    param.keepdims = args[4].operator bool();
+  }
+
+  attrs.parsed = std::move(param);
+
+  SetAttrDict<op::NumpyMomentsParam>(&attrs);
+
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+  int num_inputs = 1;
+
+  NDArray* outputs[] = {args[5].operator NDArray*()};
+  NDArray** out = (outputs[0] == nullptr) ? nullptr : outputs;
+  int num_outputs = (outputs[0] != nullptr);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+MXNET_REGISTER_API("_npi.var")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_var");
+  op::NumpyMomentsParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+
+  // parse axis
+  if (args[1].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    if (args[1].type_code() == kDLInt) {
+      param.axis = Tuple<int>(1, args[1].operator int64_t());
+    } else {
+      param.axis = Tuple<int>(args[1].operator ObjectRef());
+    }
+  }
+
+  // parse dtype
+  if (args[2].type_code() == kNull) {
+    param.dtype = dmlc::nullopt;
+  } else {
+    param.dtype = String2MXNetTypeWithBool(args[2].operator std::string());
+  }
+
+  // parse ddof
+  param.ddof = args[3].operator int();
+
+  // parse keepdims
+  if (args[4].type_code() == kNull) {
+    param.keepdims = false;
+  } else {
+    param.keepdims = args[4].operator bool();
+  }
+
+  attrs.parsed = std::move(param);
+
+  SetAttrDict<op::NumpyMomentsParam>(&attrs);
+
+  NDArray* inputs[] = {args[0].operator NDArray*()};
+  int num_inputs = 1;
+
+  NDArray* outputs[] = {args[5].operator NDArray*()};
+  NDArray** out = (outputs[0] == nullptr) ? nullptr : outputs;
+  int num_outputs = (outputs[0] != nullptr);
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+  }
+});
+
+MXNET_REGISTER_API("_npi.average")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_average");
+  op::NumpyWeightedAverageParam param;
+  nnvm::NodeAttrs attrs;
+  attrs.op = op;
+
+  // parse axis
+  if (args[2].type_code() == kNull) {
+    param.axis = dmlc::nullopt;
+  } else {
+    if (args[2].type_code() == kDLInt) {
+      param.axis = Tuple<int>(1, args[2].operator int64_t());
+    } else {
+      param.axis = Tuple<int>(args[2].operator ObjectRef());
+    }
+  }
+
+  // parse returned
+  CHECK_NE(args[3].type_code(), kNull)
+    << "returned cannot be None";
+  param.returned = args[3].operator bool();
+
+  // parse weighted
+  CHECK_NE(args[4].type_code(), kNull)
+    << "weighted cannot be None";
+  param.weighted = args[4].operator bool();
+
+  attrs.parsed = std::move(param);
+
+  SetAttrDict<op::NumpyWeightedAverageParam>(&attrs);
+
+  int num_inputs = param.weighted ? 2 : 1;
+  NDArray* outputs[] = {args[5].operator NDArray*()};
+  NDArray** out = (outputs[0] == nullptr) ? nullptr : outputs;
+  int num_outputs = (outputs[0] != nullptr);
+
+  if (param.weighted) {
+    NDArray* inputs[] = {args[0].operator NDArray*(), args[1].operator NDArray*()};
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+    if (out) {
+      *ret = PythonArg(5);
+    } else {
+      if (param.returned) {
+        *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                       NDArrayHandle(ndoutputs[1])});
+      } else {
+        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+      }
+    }
+  } else {
+    NDArray* inputs[] = {args[0].operator NDArray*()};
+    auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, out);
+    if (out) {
+      *ret = PythonArg(5);
+    } else {
+      if (param.returned) {
+        *ret = ADT(0, {NDArrayHandle(ndoutputs[0]),
+                       NDArrayHandle(ndoutputs[1])});
+      } else {
+        *ret = reinterpret_cast<mxnet::NDArray*>(ndoutputs[0]);
+      }
+    }
+  }
+});
+
+};  // namespace mxnet
diff --git a/src/api/operator/numpy/np_tensordot_op.cc b/src/api/operator/numpy/np_tensordot_op.cc
index eef58b5b3389..55c131468b12 100644
--- a/src/api/operator/numpy/np_tensordot_op.cc
+++ b/src/api/operator/numpy/np_tensordot_op.cc
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
diff --git a/src/api/operator/utils.h b/src/api/operator/utils.h
index 53e62ee7635b..8943e8058a19 100644
--- a/src/api/operator/utils.h
+++ b/src/api/operator/utils.h
@@ -56,6 +56,16 @@ void SetAttrDict(nnvm::NodeAttrs* attrs) {
   }
 }
 
+template<typename ValueType, typename T>
+Tuple<ValueType> Obj2Tuple(const runtime::ObjectRef& src) {
+  runtime::ADT adt = Downcast<runtime::ADT, runtime::ObjectRef>(src);
+  Tuple<ValueType> ret(adt.size(), 0);
+  for (size_t i = 0; i < adt.size(); ++i) {
+    ret[i] = Downcast<T, runtime::ObjectRef>(adt[i])->value;
+  }
+  return ret;
+}
+
 }  // namespace mxnet
 
 #endif  // MXNET_API_OPERATOR_UTILS_H_
diff --git a/src/operator/numpy/np_broadcast_reduce_op.h b/src/operator/numpy/np_broadcast_reduce_op.h
index 33cee78ebf80..dbef0cd96bec 100644
--- a/src/operator/numpy/np_broadcast_reduce_op.h
+++ b/src/operator/numpy/np_broadcast_reduce_op.h
@@ -67,6 +67,7 @@ struct NumpyReduceAxesParam : public dmlc::Parameter<NumpyReduceAxesParam> {
     DMLC_DECLARE_FIELD(initial).set_default(dmlc::optional<double>())
       .describe("Starting value for the sum.");
   }
+
   void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
     std::ostringstream axis_s, dtype_s, keepdims_s, initial_s;
     axis_s << axis;
@@ -447,6 +448,7 @@ inline void NumpyReduceAxesBackwardUseNone(const nnvm::NodeAttrs& attrs,
   }
 
   BroadcastComputeImpl<xpu>(attrs, ctx, inputs, req, outputs, small);
+
   if (normalize) {
     Stream<xpu> *s = ctx.get_stream<xpu>();
     MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, IType, {
@@ -498,11 +500,27 @@ struct NumpyMomentsParam : public dmlc::Parameter<NumpyMomentsParam> {
                 "precision than the default platform integer. In that case, if a is signed then "
                 "the platform integer is used while if a is unsigned then an unsigned integer of "
                 "the same precision as the platform integer is used.");
-    DMLC_DECLARE_FIELD(ddof).set_default(0)
-      .describe("Starting value for the sum.");
     DMLC_DECLARE_FIELD(keepdims).set_default(false)
       .describe("If this is set to `True`, the reduced axes are left "
                 "in the result as dimension with size one.");
+    DMLC_DECLARE_FIELD(ddof).set_default(0)
+      .describe("Starting value for the sum.");
+  }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, dtype_s, keepdims_s, ddof_s;
+    axis_s << axis;
+    keepdims_s << keepdims;
+    ddof_s << ddof;
+    (*dict)["axis"] = axis_s.str();
+    dtype_s << dtype;
+    if (dtype.has_value()) {
+      (*dict)["dtype"] = MXNetTypeWithBool2String(dtype.value());
+    } else {
+      (*dict)["dtype"] = dtype_s.str();
+    }
+    (*dict)["keepdims"] = keepdims_s.str();
+    (*dict)["ddof"] = ddof_s.str();
   }
 };
 
@@ -558,6 +576,16 @@ struct NumpyWeightedAverageParam : public dmlc::Parameter<NumpyWeightedAveragePa
       .set_default(true)
       .describe("Auxiliary flag to deal with none weights.");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, returned_s, weighted_s;
+    axis_s << axis;
+    returned_s << returned;
+    weighted_s << weighted;
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["returned"] = returned_s.str();
+    (*dict)["weighted"] = weighted_s.str();
+  }
 };
 
 inline bool NumpyWeightedAverageShape(const nnvm::NodeAttrs& attrs,
diff --git a/src/operator/numpy/np_broadcast_reduce_op_value.cc b/src/operator/numpy/np_broadcast_reduce_op_value.cc
index 026e60e8bb25..33418667dfb7 100644
--- a/src/operator/numpy/np_broadcast_reduce_op_value.cc
+++ b/src/operator/numpy/np_broadcast_reduce_op_value.cc
@@ -51,12 +51,12 @@ inline bool NumpySumType(const nnvm::NodeAttrs& attrs,
 
   if (param.dtype.has_value()) {
     if (in_attrs->at(0) == mshadow::kBool) {
-      CHECK(param.dtype.value() == mshadow::kInt32
-          || param.dtype.value() == mshadow::kInt64
-          || param.dtype.value() == mshadow::kFloat32
-          || param.dtype.value() == mshadow::kFloat64) << "Only support the following output "
-                                                         "dtypes when input dtype is bool: "
-                                                         "int32, int64, float32, float64.";
+      CHECK(param.dtype.value() == mshadow::kInt32 ||
+            param.dtype.value() == mshadow::kInt64 ||
+            param.dtype.value() == mshadow::kFloat32 ||
+            param.dtype.value() == mshadow::kFloat64)
+        << "Only support the following output dtypes when input dtype is bool: "
+           "int32, int64, float32, float64.";
     }
     TYPE_ASSIGN_CHECK(*out_attrs, 0, param.dtype.value());
   } else if (in_attrs->at(0) == mshadow::kBool) {
@@ -126,7 +126,7 @@ void TVMOpReduce(const OpContext& ctx,
 #endif  // MXNET_USE_TVM_OP
 }
 
-NNVM_REGISTER_OP(_np_sum)
+NNVM_REGISTER_OP(_npi_sum)
 .describe(R"code()code" ADD_FILELINE)
 .set_num_inputs(1)
 .set_num_outputs(1)
@@ -145,9 +145,9 @@ NNVM_REGISTER_OP(_np_sum)
     return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
   })
 .set_attr<THasDeterministicOutput>("THasDeterministicOutput", true)
-.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_np_sum"});
+.set_attr<nnvm::FGradient>("FGradient", ElemwiseGradUseNone{"_backward_npi_sum"});
 
-NNVM_REGISTER_OP(_backward_np_sum)
+NNVM_REGISTER_OP(_backward_npi_sum)
 .set_num_outputs(1)
 .set_attr_parser(ParamParser<NumpyReduceAxesParam>)
 .set_attr<nnvm::TIsBackward>("TIsBackward", true)
@@ -155,8 +155,8 @@ NNVM_REGISTER_OP(_backward_np_sum)
 .set_attr<FCompute>("FCompute<cpu>", NumpyReduceAxesBackwardUseNone<cpu>);
 
 inline bool NumpyReduceAxesNoDTypeType(const nnvm::NodeAttrs& attrs,
-                         std::vector<int> *in_attrs,
-                         std::vector<int> *out_attrs) {
+                                       std::vector<int> *in_attrs,
+                                       std::vector<int> *out_attrs) {
   CHECK_EQ(in_attrs->size(), 1U);
   CHECK_EQ(out_attrs->size(), 1U);
   TYPE_ASSIGN_CHECK(*out_attrs, 0, in_attrs->at(0));
diff --git a/src/operator/numpy/np_broadcast_reduce_op_value.cu b/src/operator/numpy/np_broadcast_reduce_op_value.cu
index 684348fcaa37..c5111c2954cd 100644
--- a/src/operator/numpy/np_broadcast_reduce_op_value.cu
+++ b/src/operator/numpy/np_broadcast_reduce_op_value.cu
@@ -26,10 +26,10 @@
 
 namespace mxnet {
 namespace op {
-NNVM_REGISTER_OP(_np_sum)
+NNVM_REGISTER_OP(_npi_sum)
 .set_attr<FCompute>("FCompute<gpu>", NumpyReduceAxesCompute<gpu, mshadow_op::sum, true>);
 
-NNVM_REGISTER_OP(_backward_np_sum)
+NNVM_REGISTER_OP(_backward_npi_sum)
 .set_attr<FCompute>("FCompute<gpu>", NumpyReduceAxesBackwardUseNone<gpu>);
 
 NNVM_REGISTER_OP(_np_max)
diff --git a/src/operator/tensor/histogram-inl.h b/src/operator/tensor/histogram-inl.h
index 7194445d7b52..29b27c6d659d 100644
--- a/src/operator/tensor/histogram-inl.h
+++ b/src/operator/tensor/histogram-inl.h
@@ -34,6 +34,8 @@
 #include <nnvm/node.h>
 #include <nnvm/op_attr_types.h>
 #include <vector>
+#include <string>
+#include <unordered_map>
 #include <type_traits>
 #include "./util/tensor_util-inl.h"
 #include "../elemwise_op_common.h"
@@ -45,22 +47,30 @@ namespace mxnet {
 namespace op {
 
 struct HistogramParam : public dmlc::Parameter<HistogramParam> {
-    dmlc::optional<int> bin_cnt;
-    dmlc::optional<mxnet::Tuple<double>> range;
-    DMLC_DECLARE_PARAMETER(HistogramParam) {
-      DMLC_DECLARE_FIELD(bin_cnt)
-        .set_default(dmlc::optional<int>())
-        .describe("Number of bins for uniform case");
-      DMLC_DECLARE_FIELD(range)
-        .set_default(dmlc::optional<mxnet::Tuple<double>>())
-        .describe("The lower and upper range of the bins. if not provided, "
-                  "range is simply (a.min(), a.max()). values outside the "
-                  "range are ignored. the first element of the range must be "
-                  "less than or equal to the second. range affects the automatic "
-                  "bin computation as well. while bin width is computed to be "
-                  "optimal based on the actual data within range, the bin count "
-                  "will fill the entire range including portions containing no data.");
-    }
+  dmlc::optional<int> bin_cnt;
+  dmlc::optional<mxnet::Tuple<double>> range;
+  DMLC_DECLARE_PARAMETER(HistogramParam) {
+    DMLC_DECLARE_FIELD(bin_cnt)
+      .set_default(dmlc::optional<int>())
+      .describe("Number of bins for uniform case");
+    DMLC_DECLARE_FIELD(range)
+      .set_default(dmlc::optional<mxnet::Tuple<double>>())
+      .describe("The lower and upper range of the bins. if not provided, "
+                "range is simply (a.min(), a.max()). values outside the "
+                "range are ignored. the first element of the range must be "
+                "less than or equal to the second. range affects the automatic "
+                "bin computation as well. while bin width is computed to be "
+                "optimal based on the actual data within range, the bin count "
+                "will fill the entire range including portions containing no data.");
+  }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream bin_cnt_s, range_s;
+    bin_cnt_s << bin_cnt;
+    range_s << range;
+    (*dict)["bin_cnt"] = bin_cnt_s.str();
+    (*dict)["range"] = range_s.str();
+  }
 };
 
 struct FillBinBoundsKernel {

From 2e57a6bea2005cad32f7e3b1da9f5511c708b7ec Mon Sep 17 00:00:00 2001
From: Yiyan66 <57363390+Yiyan66@users.noreply.github.com>
Date: Thu, 9 Apr 2020 07:59:00 +0800
Subject: [PATCH 42/44] [numpy] FFI binary bitwise ops  (#17812)

* ffi_bitwise  binary

* retrigger ci
---
 benchmark/python/ffi/benchmark_ffi.py         |  7 +++
 python/mxnet/ndarray/numpy/_op.py             | 29 ++++++---
 .../numpy/np_elemwise_broadcast_op.cc         | 59 +++++++++++++++++++
 3 files changed, 87 insertions(+), 8 deletions(-)

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index 1e911598bbb1..f0c2519754e4 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -106,6 +106,13 @@ def prepare_workloads():
     OpArgMngr.add_workload("full_like", pool['2x2'], 2)
     OpArgMngr.add_workload("zeros_like", pool['2x2'])
     OpArgMngr.add_workload("ones_like", pool['2x2'])
+    OpArgMngr.add_workload("bitwise_and", pool['2x2'].astype(int), pool['2x2'].astype(int))
+    OpArgMngr.add_workload("bitwise_xor", pool['2x2'].astype(int), pool['2x2'].astype(int))
+    OpArgMngr.add_workload("bitwise_or", pool['2x2'].astype(int), pool['2x2'].astype(int))
+    OpArgMngr.add_workload("copysign", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("arctan2", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("hypot", pool['2x2'], pool['2x2'])
+    OpArgMngr.add_workload("ldexp", pool['2x2'].astype(int), pool['2x2'].astype(int))
     OpArgMngr.add_workload("random.uniform", low=0, high=1, size=1)
     OpArgMngr.add_workload("random.exponential", scale=2, size=(2,2))
     OpArgMngr.add_workload("random.rayleigh", scale=2, size=(2,2))
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index 3074cf2b3502..d778e34f8252 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -5175,7 +5175,9 @@ def copysign(x1, x2, out=None, **kwargs):
     >>> np.copysign(a, np.arange(3)-1)
     array([-1.,  0.,  1.])
     """
-    return _ufunc_helper(x1, x2, _npi.copysign, _np.copysign, _npi.copysign_scalar, _npi.rcopysign_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.copysign(x1, x2, out=out)
+    return _api_internal.copysign(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5943,8 +5945,9 @@ def arctan2(x1, x2, out=None, **kwargs):
     >>> np.arctan2(x, y)
     array([ 1.5707964, -1.5707964])
     """
-    return _ufunc_helper(x1, x2, _npi.arctan2, _np.arctan2,
-                         _npi.arctan2_scalar, _npi.rarctan2_scalar, out=out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.arctan2(x1, x2, out=out)
+    return _api_internal.arctan2(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -5992,7 +5995,9 @@ def hypot(x1, x2, out=None, **kwargs):
            [ 5.,  5.,  5.],
            [ 5.,  5.,  5.]])
     """
-    return _ufunc_helper(x1, x2, _npi.hypot, _np.hypot, _npi.hypot_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.hypot(x1, x2, out=out)
+    return _api_internal.hypot(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6032,7 +6037,9 @@ def bitwise_and(x1, x2, out=None, **kwargs):
     >>> np.bitwise_and(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))
     array([False,  True])
     """
-    return _ufunc_helper(x1, x2, _npi.bitwise_and, _np.bitwise_and, _npi.bitwise_and_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.bitwise_and(x1, x2, out=out)
+    return _api_internal.bitwise_and(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6070,7 +6077,9 @@ def bitwise_xor(x1, x2, out=None, **kwargs):
     >>> np.bitwise_xor(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))
     array([ True, False])
     """
-    return _ufunc_helper(x1, x2, _npi.bitwise_xor, _np.bitwise_xor, _npi.bitwise_xor_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.bitwise_xor(x1, x2, out=out)
+    return _api_internal.bitwise_xor(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6108,7 +6117,9 @@ def bitwise_or(x1, x2, out=None, **kwargs):
     >>> np.bitwise_or(np.array([True, True], dtype='bool'), np.array([False, True], dtype='bool'))
     array([ True, True])
     """
-    return _ufunc_helper(x1, x2, _npi.bitwise_or, _np.bitwise_or, _npi.bitwise_or_scalar, None, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.bitwise_or(x1, x2, out=out)
+    return _api_internal.bitwise_or(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
@@ -6147,7 +6158,9 @@ def ldexp(x1, x2, out=None, **kwargs):
     >>> np.ldexp(5, np.arange(4))
     array([  5.,  10.,  20.,  40.])
     """
-    return _ufunc_helper(x1, x2, _npi.ldexp, _np.ldexp, _npi.ldexp_scalar, _npi.rldexp_scalar, out)
+    if isinstance(x1, numeric_types) and isinstance(x2, numeric_types):
+        return _np.ldexp(x1, x2, out=out)
+    return _api_internal.ldexp(x1, x2, out)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/src/api/operator/numpy/np_elemwise_broadcast_op.cc b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
index 7a9eb0139439..00c350be0daf 100644
--- a/src/api/operator/numpy/np_elemwise_broadcast_op.cc
+++ b/src/api/operator/numpy/np_elemwise_broadcast_op.cc
@@ -88,4 +88,63 @@ MXNET_REGISTER_API("_npi.lcm")
   UFuncHelper(args, ret, op, op_scalar, nullptr);
 });
 
+MXNET_REGISTER_API("_npi.bitwise_or")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_bitwise_or");
+  const nnvm::Op* op_scalar = Op::Get("_npi_bitwise_or_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.bitwise_xor")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_bitwise_xor");
+  const nnvm::Op* op_scalar = Op::Get("_npi_bitwise_xor_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.bitwise_and")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_bitwise_and");
+  const nnvm::Op* op_scalar = Op::Get("_npi_bitwise_and_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.copysign")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_copysign");
+  const nnvm::Op* op_scalar = Op::Get("_npi_copysign_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rcopysign_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.arctan2")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_arctan2");
+  const nnvm::Op* op_scalar = Op::Get("_npi_arctan2_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rarctan2_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
+MXNET_REGISTER_API("_npi.hypot")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_hypot");
+  const nnvm::Op* op_scalar = Op::Get("_npi_hypot_scalar");
+  UFuncHelper(args, ret, op, op_scalar, nullptr);
+});
+
+MXNET_REGISTER_API("_npi.ldexp")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_ldexp");
+  const nnvm::Op* op_scalar = Op::Get("_npi_ldexp_scalar");
+  const nnvm::Op* op_rscalar = Op::Get("_npi_rldexp_scalar");
+  UFuncHelper(args, ret, op, op_scalar, op_rscalar);
+});
+
 }  // namespace mxnet

From 1944c706b25f101b899afa6d3f00a1d44a2926da Mon Sep 17 00:00:00 2001
From: AntiZpvoh <59728467+AntiZpvoh@users.noreply.github.com>
Date: Tue, 14 Apr 2020 06:35:32 +0800
Subject: [PATCH 43/44] [Numpy] FFI: random.choice, take and clip (#17854)

* change the header file of np.random.choice

* add np_choice_op.cc file

* add including header file

* implement the basic function of random.choice

* try to use take op in backend

* try to use take op in backend

* add take invoking function

* fix some syntax problems

* fix some problems

* complete numpy.random.choice ffi

* first commit of ffi indexing_op.cc

* add random.choice ffi benchmark

* complete take ffi

* change the implementation of random.choice

* add take op benchmark

* complete clip op ffi and fix a problem

* add clip op benchmark

* fix some sanity problems

* add space before ( and fix reimport

* fix a typo

* remove dead code and remove new operator

Co-authored-by: Ubuntu <ubuntu@ip-172-31-15-23.us-east-2.compute.internal>
---
 benchmark/python/ffi/benchmark_ffi.py         |  3 +
 python/mxnet/ndarray/numpy/_op.py             | 10 +--
 python/mxnet/ndarray/numpy/random.py          | 22 ++---
 src/api/operator/numpy/random/np_choice_op.cc | 89 +++++++++++++++++++
 .../operator/numpy/random/np_laplace_op.cc    |  2 +-
 src/api/operator/tensor/indexing_op.cc        | 78 ++++++++++++++++
 src/api/operator/tensor/matrix_op.cc          | 71 +++++++++++++++
 src/operator/numpy/random/np_choice_op.h      | 11 +++
 src/operator/tensor/indexing_op.h             | 21 +++++
 src/operator/tensor/matrix_op-inl.h           |  8 ++
 10 files changed, 292 insertions(+), 23 deletions(-)
 create mode 100644 src/api/operator/numpy/random/np_choice_op.cc
 create mode 100644 src/api/operator/tensor/indexing_op.cc
 create mode 100644 src/api/operator/tensor/matrix_op.cc

diff --git a/benchmark/python/ffi/benchmark_ffi.py b/benchmark/python/ffi/benchmark_ffi.py
index f0c2519754e4..1f30551a5598 100644
--- a/benchmark/python/ffi/benchmark_ffi.py
+++ b/benchmark/python/ffi/benchmark_ffi.py
@@ -101,6 +101,9 @@ def prepare_workloads():
     OpArgMngr.add_workload("diff", pool['2x2'], n=1, axis=-1)
     OpArgMngr.add_workload("nonzero", pool['2x2'])
     OpArgMngr.add_workload("tril", pool['2x2'], k=0)
+    OpArgMngr.add_workload("random.choice", pool['2'], size=(2, 2))
+    OpArgMngr.add_workload("take", pool['2'], dnp.array([1,0], dtype='int64'))
+    OpArgMngr.add_workload("clip", pool['2x2'], 0, 1)
     OpArgMngr.add_workload("expand_dims", pool['2x2'], axis=0)
     OpArgMngr.add_workload("broadcast_to", pool['2x2'], (2, 2, 2))
     OpArgMngr.add_workload("full_like", pool['2x2'], 2)
diff --git a/python/mxnet/ndarray/numpy/_op.py b/python/mxnet/ndarray/numpy/_op.py
index d778e34f8252..025e14cd8bf3 100644
--- a/python/mxnet/ndarray/numpy/_op.py
+++ b/python/mxnet/ndarray/numpy/_op.py
@@ -690,9 +690,9 @@ def take(a, indices, axis=None, mode='raise', out=None):
         raise NotImplementedError(
             "function take does not support mode '{}'".format(mode))
     if axis is None:
-        return _npi.take(_npi.reshape(a, -1), indices, 0, mode, out)
+        return _api_internal.take(_npi.reshape(a, -1), indices, 0, mode, out)
     else:
-        return _npi.take(a, indices, axis, mode, out)
+        return _api_internal.take(a, indices, axis, mode, out)
 # pylint: enable=redefined-outer-name
 
 
@@ -4548,11 +4548,7 @@ def clip(a, a_min, a_max, out=None):
     """
     if a_min is None and a_max is None:
         raise ValueError('array_clip: must set either max or min')
-    if a_min is None:
-        a_min = float('-inf')
-    if a_max is None:
-        a_max = float('inf')
-    return _npi.clip(a, a_min, a_max, out=out)
+    return _api_internal.clip(a, a_min, a_max, out)
 
 
 @set_module('mxnet.ndarray.numpy')
diff --git a/python/mxnet/ndarray/numpy/random.py b/python/mxnet/ndarray/numpy/random.py
index a25096a65224..1f4cd2597825 100644
--- a/python/mxnet/ndarray/numpy/random.py
+++ b/python/mxnet/ndarray/numpy/random.py
@@ -509,24 +509,16 @@ def choice(a, size=None, replace=True, p=None, ctx=None, out=None):
     """
     from ...numpy import ndarray as np_ndarray
     if ctx is None:
-        ctx = current_context()
+        ctx = str(current_context())
+    else:
+        ctx = str(ctx)
     if size == ():
         size = None
     if isinstance(a, np_ndarray):
-        ctx = None
-        if p is None:
-            indices = _npi.choice(a, a=None, size=size,
-                                  replace=replace, ctx=ctx, weighted=False)
-            return _npi.take(a, indices)
-        else:
-            indices = _npi.choice(a, p, a=None, size=size,
-                                  replace=replace, ctx=ctx, weighted=True)
-            return _npi.take(a, indices)
+        indices = _api_internal.choice(a, size, replace, p, ctx, out)
+        return _api_internal.take(a, indices, 0, 'raise', out)
     else:
-        if p is None:
-            return _npi.choice(a=a, size=size, replace=replace, ctx=ctx, weighted=False, out=out)
-        else:
-            return _npi.choice(p, a=a, size=size, replace=replace, ctx=ctx, weighted=True, out=out)
+        return _api_internal.choice(a, size, replace, p, ctx, out)
 
 
 def exponential(scale=1.0, size=None, ctx=None, out=None):
@@ -793,7 +785,7 @@ def beta(a, b, size=None, dtype=None, ctx=None):
     # use fp64 to prevent precision loss
     X = gamma(a, 1, size=size, dtype='float64', ctx=ctx)
     Y = gamma(b, 1, size=size, dtype='float64', ctx=ctx)
-    out = X/(X + Y)
+    out = X / (X + Y)
     return out.astype(dtype)
 
 
diff --git a/src/api/operator/numpy/random/np_choice_op.cc b/src/api/operator/numpy/random/np_choice_op.cc
new file mode 100644
index 000000000000..fe7b54d512c8
--- /dev/null
+++ b/src/api/operator/numpy/random/np_choice_op.cc
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file np_choice_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/np_choice_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../../utils.h"
+#include "../../../../operator/numpy/random/np_choice_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.choice")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_choice");
+  nnvm::NodeAttrs attrs;
+  op::NumpyChoiceParam param;
+
+  NDArray* inputs[2];
+  int num_inputs = 0;
+
+  if (args[0].type_code() == kDLInt) {
+    param.a = args[0].operator int();
+  } else if (args[0].type_code() == kNDArrayHandle) {
+    param.a = dmlc::nullopt;
+    inputs[num_inputs] = args[0].operator mxnet::NDArray*();
+    num_inputs++;
+  }
+
+  if (args[1].type_code() == kNull) {
+    param.size = dmlc::nullopt;
+  } else {
+    if (args[1].type_code() == kDLInt) {
+      param.size = mxnet::Tuple<int64_t>(1, args[1].operator int64_t());
+    } else {
+      param.size = mxnet::Tuple<int64_t>(args[1].operator ObjectRef());
+    }
+  }
+
+  if (args[2].type_code() == kNull) {
+    param.replace = true;
+  } else {
+    param.replace = args[2].operator bool();
+  }
+
+  if (args[3].type_code() == kNull) {
+    param.weighted = false;
+  } else if (args[0].type_code() == kNDArrayHandle) {
+    param.weighted = true;
+    inputs[num_inputs] = args[3].operator mxnet::NDArray*();
+    num_inputs++;
+  }
+
+  attrs.parsed = std::move(param);
+  attrs.op = op;
+  if (args[4].type_code() != kNull) {
+    attrs.dict["ctx"] = args[4].operator std::string();
+  }
+  NDArray* out = args[5].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, num_inputs, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(5);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/numpy/random/np_laplace_op.cc b/src/api/operator/numpy/random/np_laplace_op.cc
index 40e79017c0f2..57f770bfa376 100644
--- a/src/api/operator/numpy/random/np_laplace_op.cc
+++ b/src/api/operator/numpy/random/np_laplace_op.cc
@@ -19,7 +19,7 @@
 
 /*!
  * \file np_laplace_op.cc
- * \brief Implementation of the API of functions in src/operator/numpy/np_laplace_op.cc
+ * \brief Implementation of the API of functions in src/operator/numpy/random/np_laplace_op.cc
  */
 #include <mxnet/api_registry.h>
 #include <mxnet/runtime/packed_func.h>
diff --git a/src/api/operator/tensor/indexing_op.cc b/src/api/operator/tensor/indexing_op.cc
new file mode 100644
index 000000000000..df194018c712
--- /dev/null
+++ b/src/api/operator/tensor/indexing_op.cc
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file indexing_op.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/indexing_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/tensor/indexing_op.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.take")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_take");
+  nnvm::NodeAttrs attrs;
+  op::TakeParam param;
+  NDArray* inputs[2];
+
+  if (args[0].type_code() != kNull) {
+    inputs[0] = args[0].operator mxnet::NDArray *();
+  }
+
+  if (args[1].type_code() != kNull) {
+    inputs[1] = args[1].operator mxnet::NDArray *();
+  }
+
+  if (args[2].type_code() == kDLInt) {
+    param.axis = args[2].operator int();
+  }
+
+  if (args[3].type_code() != kNull) {
+    std::string mode = args[3].operator std::string();
+    if (mode == "raise") {
+      param.mode = op::take_::kRaise;
+    } else if (mode == "clip") {
+      param.mode = op::take_::kClip;
+    } else if (mode == "wrap") {
+      param.mode = op::take_::kWrap;
+    }
+  }
+
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::TakeParam>(&attrs);
+
+  NDArray* out = args[4].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  // set the number of outputs provided by the `out` arugment
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, 2, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(4);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/api/operator/tensor/matrix_op.cc b/src/api/operator/tensor/matrix_op.cc
new file mode 100644
index 000000000000..ed91b091cc39
--- /dev/null
+++ b/src/api/operator/tensor/matrix_op.cc
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file matrix_op.cc
+ * \brief Implementation of the API of functions in src/operator/tensor/matrix_op.cc
+ */
+#include <mxnet/api_registry.h>
+#include <mxnet/runtime/packed_func.h>
+#include "../utils.h"
+#include "../../../operator/tensor/matrix_op-inl.h"
+
+namespace mxnet {
+
+MXNET_REGISTER_API("_npi.clip")
+.set_body([](runtime::MXNetArgs args, runtime::MXNetRetValue* ret) {
+  using namespace runtime;
+  const nnvm::Op* op = Op::Get("_npi_clip");
+  nnvm::NodeAttrs attrs;
+  op::ClipParam param;
+  NDArray* inputs[1];
+
+  if (args[0].type_code() != kNull) {
+    inputs[0] = args[0].operator mxnet::NDArray *();
+  }
+
+  if (args[1].type_code() != kNull) {
+    param.a_min = args[1].operator double();
+  } else {
+    param.a_min = -INFINITY;
+  }
+
+  if (args[2].type_code() != kNull) {
+    param.a_max = args[2].operator double();
+  } else {
+    param.a_max = INFINITY;
+  }
+
+  attrs.parsed = param;
+  attrs.op = op;
+  SetAttrDict<op::ClipParam>(&attrs);
+
+  NDArray* out = args[3].operator mxnet::NDArray*();
+  NDArray** outputs = out == nullptr ? nullptr : &out;
+  // set the number of outputs provided by the `out` arugment
+  int num_outputs = out != nullptr;
+  auto ndoutputs = Invoke(op, &attrs, 1, inputs, &num_outputs, outputs);
+  if (out) {
+    *ret = PythonArg(3);
+  } else {
+    *ret = ndoutputs[0];
+  }
+});
+
+}  // namespace mxnet
diff --git a/src/operator/numpy/random/np_choice_op.h b/src/operator/numpy/random/np_choice_op.h
index a6a7cecfefd5..bc1e712aeba0 100644
--- a/src/operator/numpy/random/np_choice_op.h
+++ b/src/operator/numpy/random/np_choice_op.h
@@ -53,6 +53,17 @@ struct NumpyChoiceParam : public dmlc::Parameter<NumpyChoiceParam> {
     DMLC_DECLARE_FIELD(replace).set_default(true);
     DMLC_DECLARE_FIELD(weighted).set_default(false);
   }
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_s, size_s, replace_s, weighted_s;
+    a_s << a;
+    size_s << size;
+    replace_s << replace;
+    weighted_s << weighted;
+    (*dict)["a"] = a_s.str();
+    (*dict)["size"] = size_s.str();
+    (*dict)["replace"] = replace_s.str();
+    (*dict)["weighted"] = weighted_s.str();
+  }
 };
 
 inline bool NumpyChoiceOpType(const nnvm::NodeAttrs &attrs,
diff --git a/src/operator/tensor/indexing_op.h b/src/operator/tensor/indexing_op.h
index 2b048813a464..cd85daa80df3 100644
--- a/src/operator/tensor/indexing_op.h
+++ b/src/operator/tensor/indexing_op.h
@@ -680,6 +680,27 @@ struct TakeParam: public dmlc::Parameter<TakeParam> {
               " \"wrap\" means to wrap around."
               " \"raise\" means to raise an error when index out of range.");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream axis_s, mode_s;
+    axis_s << axis;
+    mode_s << mode;
+    (*dict)["axis"] = axis_s.str();
+    (*dict)["mode"] = mode_s.str();
+    switch (mode) {
+      case take_::kRaise:
+        (*dict)["mode"] = "raise";
+        break;
+      case take_::kClip:
+        (*dict)["mode"] = "clip";
+        break;
+      case take_::kWrap:
+        (*dict)["mode"] = "wrap";
+        break;
+      default:
+        (*dict)["mode"] = mode_s.str();
+    }
+  }
 };
 
 inline bool TakeOpShape(const nnvm::NodeAttrs& attrs,
diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h
index 6efde79f202b..821fa8587081 100644
--- a/src/operator/tensor/matrix_op-inl.h
+++ b/src/operator/tensor/matrix_op-inl.h
@@ -1605,6 +1605,14 @@ struct ClipParam : public dmlc::Parameter<ClipParam> {
     DMLC_DECLARE_FIELD(a_max)
     .describe("Maximum value");
   }
+
+  void SetAttrDict(std::unordered_map<std::string, std::string>* dict) {
+    std::ostringstream a_min_s, a_max_s;
+    a_min_s << a_min;
+    a_max_s << a_max;
+    (*dict)["a_min"] = a_min_s.str();
+    (*dict)["a_max"] = a_max_s.str();
+  }
 };
 
 

From 327f7ad1612acadf3152d385ebb199bb12c4b110 Mon Sep 17 00:00:00 2001
From: Xi Wang <xidulu@gmail.com>
Date: Fri, 13 Mar 2020 02:14:28 +0800
Subject: [PATCH 44/44] fix np.clip scalar input case (#17788)

---
 python/mxnet/numpy/multiarray.py       |  5 +++++
 tests/python/unittest/test_numpy_op.py | 10 ++++++++++
 2 files changed, 15 insertions(+)

diff --git a/python/mxnet/numpy/multiarray.py b/python/mxnet/numpy/multiarray.py
index d8f7f4a29ce3..fd60d1a05dc7 100644
--- a/python/mxnet/numpy/multiarray.py
+++ b/python/mxnet/numpy/multiarray.py
@@ -6490,6 +6490,11 @@ def clip(a, a_min, a_max, out=None):
     >>> np.clip(a, 3, 6, out=a)
     array([3., 3., 3., 3., 4., 5., 6., 6., 6., 6.], dtype=float32)
     """
+    from numbers import Number
+    if isinstance(a, Number):
+        # In case input is a scalar, the computation would fall back to native numpy.
+        # The value returned would be a python scalar.
+        return _np.clip(a, a_min, a_max, out=None)
     return _mx_nd_np.clip(a, a_min, a_max, out=out)
 
 
diff --git a/tests/python/unittest/test_numpy_op.py b/tests/python/unittest/test_numpy_op.py
index 88917aac0aca..bc7c07562f62 100644
--- a/tests/python/unittest/test_numpy_op.py
+++ b/tests/python/unittest/test_numpy_op.py
@@ -3809,6 +3809,16 @@ def __init__(self, a_min=None, a_max=None):
 
         def hybrid_forward(self, F, x):
             return x.clip(self._a_min, self._a_max)
+    
+    # Test scalar case
+    for _, a_min, a_max, throw_exception in workloads:
+        a = _np.random.uniform() # A scalar
+        if throw_exception:
+            # No need to test the exception case here.
+            continue
+        mx_ret = np.clip(a, a_min, a_max)
+        np_ret = _np.clip(a, a_min, a_max)
+        assert_almost_equal(mx_ret, np_ret, atol=1e-4, rtol=1e-3, use_broadcast=False)
 
     for shape, a_min, a_max, throw_exception in workloads:
         for dtype in dtypes: