From 2d7c7b16168e93e989be48d57db37589233e9c1d Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Wed, 17 May 2023 20:37:55 +0800
Subject: [PATCH 1/6] add more parameter for softplus/gelu paddle op

---
 python/tvm/relay/frontend/paddlepaddle.py     | 29 ++++++++++++++++---
 .../frontend/paddlepaddle/test_forward.py     | 28 +++++++++++++++---
 2 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py
index adff28187e5b..4f280ac12b04 100755
--- a/python/tvm/relay/frontend/paddlepaddle.py
+++ b/python/tvm/relay/frontend/paddlepaddle.py
@@ -833,10 +833,28 @@ def convert_gelu(g, op, block):
     """Operator converter for gelu."""
 
     x = g.get_node(op.input("X")[0])
-    out = x * (
-        _expr.const(0.5, dtype="float32")
-        + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32")
-    )
+    approx = op.attr("approximate")
+    if approx:
+        M_2_SQRTPI = 1.12837916709551257390
+        M_SQRT1_2 = 0.70710678118654752440
+        out = x * (
+            _expr.const(0.5, dtype="float32")
+            + _op.tanh(
+                x
+                * _expr.const(M_2_SQRTPI * M_SQRT1_2, dtype="float32")
+                * (
+                    _expr.const(1.0, dtype="float32")
+                    + _expr.const((0.044715), dtype="float32") * x * x
+                )
+            )
+            * _expr.const(0.5, dtype="float32")
+        )
+    else:
+        out = x * (
+            _expr.const(0.5, dtype="float32")
+            + _op.erf(x * _expr.const(0.5**0.5, dtype="float32"))
+            * _expr.const(0.5, dtype="float32")
+        )
     g.add_node(op.output("Out")[0], out)
 
 
@@ -2172,8 +2190,11 @@ def convert_softplus(g, op, block):
     x = g.get_node(op.input("X")[0])
     dtype = infer_type(x).checked_type.dtype
     beta = op.attr("beta")
+    threshold = op.attr("threshold")
     beta = _expr.const(beta, dtype=dtype)
+    threshold = _expr.const(threshold, dtype=dtype)
     out = _op.log(_op.exp(x * beta) + _expr.const(1.0, dtype=dtype)) / beta
+    out = _op.where(x * beta > threshold, x, out)  # revert to linear if x * beta > threshold
     g.add_node(op.output("Out")[0], out)
 
 
diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py
index 289fc0faa3ef..a26a8262ef0a 100755
--- a/tests/python/frontend/paddlepaddle/test_forward.py
+++ b/tests/python/frontend/paddlepaddle/test_forward.py
@@ -1005,13 +1005,18 @@ def ones_like2(inputs):
 
 @tvm.testing.uses_gpu
 def test_forward_gelu():
-    @paddle.jit.to_static
-    def gelu(inputs):
-        return nn.functional.gelu(inputs)
+    class Gelu(nn.Layer):
+        def __init__(self, approximate=False):
+            super(Gelu, self).__init__()
+            self.approximate = approximate
+
+        def forward(self, inputs):
+            return nn.functional.gelu(inputs, approximate=self.approximate)
 
     input_shape = [1, 3, 10, 10]
     input_data = paddle.rand(input_shape, dtype="float32")
-    verify_model(gelu, input_data=input_data)
+    verify_model(Gelu(), input_data=input_data)
+    verify_model(Gelu(approximate=True), input_data=input_data)
 
 
 @tvm.testing.uses_gpu
@@ -1770,6 +1775,21 @@ def test_forward_tanh():
     pass
 
 
+@tvm.testing.uses_gpu
+def test_forward_softplusThreshold():
+    class Softplus(nn.Layer):
+        def __init__(self, threshold=20):
+            super(Softplus, self).__init__()
+            self.threshold = threshold
+
+        def forward(self, input):
+            return paddle.nn.functional.softplus(input, threshold=self.threshold)
+
+    input_data = paddle.rand([2, 3, 5], dtype="float32")
+    verify_model(Softplus(), input_data=input_data)
+    verify_model(Softplus(threshold=0.5), input_data=input_data)
+
+
 @tvm.testing.uses_gpu
 def test_forward_meshgrid():
     @paddle.jit.to_static

From 9f8e603f73ddd23e812254b4e8487a318657f8b6 Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Wed, 17 May 2023 20:47:53 +0800
Subject: [PATCH 2/6] merge

---
 python/tvm/relay/frontend/paddlepaddle.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py
index 3124c45b617f..670a9d7aa55a 100755
--- a/python/tvm/relay/frontend/paddlepaddle.py
+++ b/python/tvm/relay/frontend/paddlepaddle.py
@@ -2259,7 +2259,6 @@ def convert_softplus(g, op, block):
     x = g.get_node(op.input("X")[0])
     dtype = infer_type(x).checked_type.dtype
     beta = op.attr("beta")
-    threshold = op.attr("threshold")
     beta = _expr.const(beta, dtype=dtype)
     threshold = op.attr("threshold")
     threshold = _expr.const(threshold, dtype=dtype)

From 8d6dbd49abebc4250266e0565be32df0c1fb8762 Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Mon, 29 May 2023 17:21:32 +0800
Subject: [PATCH 3/6] merge

---
 .../python/frontend/paddlepaddle/test_forward.py  | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py
index c6d04f05caf1..d45c350921cc 100755
--- a/tests/python/frontend/paddlepaddle/test_forward.py
+++ b/tests/python/frontend/paddlepaddle/test_forward.py
@@ -1777,21 +1777,6 @@ def test_forward_tanh():
     pass
 
 
-@tvm.testing.uses_gpu
-def test_forward_softplusThreshold():
-    class Softplus(nn.Layer):
-        def __init__(self, threshold=20):
-            super(Softplus, self).__init__()
-            self.threshold = threshold
-
-        def forward(self, input):
-            return paddle.nn.functional.softplus(input, threshold=self.threshold)
-
-    input_data = paddle.rand([2, 3, 5], dtype="float32")
-    verify_model(Softplus(), input_data=input_data)
-    verify_model(Softplus(threshold=0.5), input_data=input_data)
-
-
 @tvm.testing.uses_gpu
 def test_forward_meshgrid():
     @paddle.jit.to_static

From 77f6d24f71eaccfaa9bddac65834ac8b1682ef99 Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Mon, 29 May 2023 17:24:27 +0800
Subject: [PATCH 4/6] add tanhshrink op

---
 python/tvm/relay/frontend/paddlepaddle.py          |  9 +++++++++
 tests/python/frontend/paddlepaddle/test_forward.py | 10 ++++++++++
 2 files changed, 19 insertions(+)

diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py
index 670a9d7aa55a..38d66e9232e7 100755
--- a/python/tvm/relay/frontend/paddlepaddle.py
+++ b/python/tvm/relay/frontend/paddlepaddle.py
@@ -2386,6 +2386,14 @@ def convert_take_along_axis(g, op, block):
     g.add_node(op.output("Result")[0], out)
 
 
+def convert_tanhshrink(g, op, block):
+    """Operator converter for tanhshrink."""
+
+    x = g.get_node(op.input("X")[0])
+    out = x - _op.tanh(x)
+    g.add_node(op.output("Out")[0], out)
+
+
 def convert_thresholded_relu(g, op, block):
     """Operator converter for thresholded_relu."""
 
@@ -2697,6 +2705,7 @@ def convert_where_index(g, op, block):
     "take_along_axis": convert_take_along_axis,
     "tan": convert_unary_op,
     "tanh": convert_unary_op,
+    "tanh_shrink": convert_tanhshrink,
     "top_k": convert_topk,
     "thresholded_relu": convert_thresholded_relu,
     "tile": convert_tile,
diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py
index d45c350921cc..3f89b4d94bed 100755
--- a/tests/python/frontend/paddlepaddle/test_forward.py
+++ b/tests/python/frontend/paddlepaddle/test_forward.py
@@ -1936,6 +1936,16 @@ def topk8(inputs):
     verify_model(topk8, input_data=input_data_fp32)
 
 
+@tvm.testing.uses_gpu
+def test_forward_tanhshrink():
+    @paddle.jit.to_static
+    def tanhshrink(inputs):
+        return paddle.nn.functional.tanhshrink(inputs)
+
+    input_data = paddle.randn(shape=[2, 3], dtype="float32")
+    verify_model(tanhshrink, input_data=input_data)
+
+
 @tvm.testing.uses_gpu
 def test_forward_one_hot_v2():
     @paddle.jit.to_static

From cce615509fe788c8b03f85bb0e51f5ef28d5a031 Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Wed, 31 May 2023 19:26:22 +0800
Subject: [PATCH 5/6] add pool3d and set_value

---
 python/tvm/relay/frontend/paddlepaddle.py     | 168 ++++++++++++++++++
 .../frontend/paddlepaddle/test_forward.py     |  81 +++++++++
 2 files changed, 249 insertions(+)

diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py
index 38d66e9232e7..c35330329bbe 100755
--- a/python/tvm/relay/frontend/paddlepaddle.py
+++ b/python/tvm/relay/frontend/paddlepaddle.py
@@ -1548,6 +1548,105 @@ def convert_pool2d(g, op, block):
     g.add_node(op.output("Out")[0], out)
 
 
+def convert_pool3d(g, op, block):
+    """Operator converter for pool3d."""
+
+    adaptive = op.attr("adaptive")
+    ceil_mode = op.attr("ceil_mode")
+    global_pooling = op.attr("global_pooling")
+    ksize = op.attr("ksize")
+    paddings = op.attr("paddings")
+    padding_algorithm = op.attr("padding_algorithm")
+    pooling_type = op.attr("pooling_type")
+    data_format = op.attr("data_format")
+
+    if global_pooling:
+        adaptive = True
+        ksize = [1, 1, 1]
+
+    input_x = g.get_node(op.input("X")[0])
+    _, _, _, in_h, in_w = infer_shape(input_x)
+
+    op_map = {
+        "avg": "avg_pool3d",
+        "max": "max_pool3d",
+    }
+
+    strides = op.attr("strides")
+    if isinstance(strides, int):
+        strides = [strides, strides]
+    if isinstance(ksize, int):
+        ksize = [ksize, ksize, ksize]
+    if isinstance(paddings, int):
+        paddings = [paddings] * 3
+
+    if padding_algorithm == "VALID":
+        paddings = [0, 0, 0]
+    elif padding_algorithm == "SAME":
+        input_x = autopad(input_x, strides, ksize)
+        paddings = [0, 0, 0]
+    elif padding_algorithm == "EXPLICIT":
+        if len(paddings) == 3:
+            paddings = [
+                paddings[0],
+                paddings[1],
+                paddings[2],
+                paddings[0],
+                paddings[1],
+                paddings[2],
+            ]
+        elif len(paddings) == 6:
+            paddings = [
+                paddings[0],
+                paddings[3],
+                paddings[1],
+                paddings[4],
+                paddings[2],
+                paddings[5],
+            ]
+    else:
+        msg = 'Value {} in attribute "padding" of operator Pool3d is not "valid."'
+        raise tvm.error.OpAttributeInvalid(msg.format(padding_algorithm))
+
+    # handle with special case
+    # while kernel size less than input size
+    # shrink kernel size to input size
+    if (
+        not isinstance(in_h, _op.Expr)
+        and padding_algorithm == "EXPLICIT"
+        and in_h + paddings[0] + paddings[2] < ksize[0]
+    ):
+        ksize[0] = in_h
+    if (
+        not isinstance(in_w, _op.Expr)
+        and padding_algorithm == "EXPLICIT"
+        and in_w + paddings[1] + paddings[3] < ksize[1]
+    ):
+        ksize[1] = in_w
+
+    if not adaptive:
+        if pooling_type == "avg":
+            exclusive = op.attr("exclusive")
+            out = _op.nn.avg_pool3d(
+                input_x,
+                pool_size=ksize,
+                strides=strides,
+                padding=paddings,
+                ceil_mode=ceil_mode,
+                count_include_pad=not exclusive,
+                layout=data_format,
+            )
+        else:
+            out = getattr(_op.nn, op_map[pooling_type])(
+                input_x, pool_size=ksize, strides=strides, padding=paddings, ceil_mode=ceil_mode
+            )
+    else:
+        out = getattr(_op.nn, "adaptive_" + op_map[pooling_type])(
+            input_x, output_size=ksize, layout=data_format
+        )
+    g.add_node(op.output("Out")[0], out)
+
+
 def convert_pow(g, op, block):
     """Operator converter for pow."""
 
@@ -2056,6 +2155,73 @@ def convert_selu(g, op, block):
     g.add_node(op.output("Out")[0], out)
 
 
+def convert_set_value(g, op, block):
+    """Operator converter for set_value."""
+
+    x = g.get_node(op.input("Input")[0])
+    if op.input("StartsTensorList"):
+        starts = g.get_node(op.input("StartsTensorList")[0])
+    else:
+        starts = op.attr("starts")[0]
+
+    if op.input("EndsTensorList"):
+        ends = g.get_node(op.input("EndsTensorList")[0])
+    else:
+        ends = op.attr("ends")[0]
+
+    axes = op.attr("axes")
+    assert len(axes) == 1, "Only support one axes now."
+    axes = axes[0]
+
+    input_shape = infer_shape(x)
+    ends = min(ends, input_shape[axes])
+
+    if op.input("StepsTensorList"):
+        steps = g.get_node(op.input("StepsTensorList")[0])
+    else:
+        steps = op.attr("steps")[0]
+
+    if op.input("ValueTensor"):
+        value = g.get_node(op.input("ValueTensor")[0])
+    else:
+        input_dtype = infer_type(x).checked_type.dtype
+        if input_dtype == "float64":
+            value = _expr.const(op.attr("fp64_values"), dtype="float64")
+        elif input_dtype == "float32":
+            value = _expr.const(op.attr("fp32_values"), dtype="float32")
+        elif input_dtype == "int32":
+            value = _expr.const(op.attr("int32_values"), dtype="int32")
+        elif input_dtype == "int64":
+            value = _expr.const(op.attr("int64_values"), dtype="int64")
+        else:
+            raise tvm.error.OpNotImplemented(
+                "dtype {} is not supported for set_value".format(input_dtype)
+            )
+
+    sliced_data = _op.strided_slice(x, begin=[starts], end=[ends], strides=[steps], axes=[axes])
+    sliced_shape = infer_shape(sliced_data)
+
+    if infer_shape(value) != sliced_shape:
+        expand_value = _op.broadcast_to(value, sliced_shape)
+    else:
+        expand_value = value
+
+    if starts < 0:
+        starts = starts + input_shape[axes]
+    if ends < 0:
+        ends = ends + input_shape[axes]
+
+    indices = _op.arange(
+        start=_expr.const(starts, dtype="int32"),
+        stop=_expr.const(ends, dtype="int32"),
+        step=_expr.const(steps, dtype="int32"),
+        dtype="int32",
+    )
+    indices = _op.expand_dims(indices, axis=0)
+    out = _op.scatter_nd(x, indices, expand_value, "update")
+    g.add_node(op.output("Out")[0], out)
+
+
 def convert_shape(g, op, block):
     """Operator converter for shape."""
 
@@ -2660,6 +2826,7 @@ def convert_where_index(g, op, block):
     "pad3d": convert_padding,
     "pixel_shuffle": convert_pixel_shuffle,
     "pool2d": convert_pool2d,
+    "pool3d": convert_pool3d,
     "pow": convert_pow,
     "prelu": convert_prelu,
     "range": convert_range,
@@ -2682,6 +2849,7 @@ def convert_where_index(g, op, block):
     "scatter": convert_scatter,
     "scatter_nd_add": convert_scatter_nd_add,
     "selu": convert_selu,
+    "set_value": convert_set_value,
     "shape": convert_shape,
     "sigmoid": convert_unary_op,
     "sign": convert_unary_op,
diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py
index 3f89b4d94bed..2b1b2a1cc75a 100755
--- a/tests/python/frontend/paddlepaddle/test_forward.py
+++ b/tests/python/frontend/paddlepaddle/test_forward.py
@@ -2402,5 +2402,86 @@ def forward(self, input_data, label):
     verify_model(SoftmaxWithCrossEntropy(soft_label=True, axis=0), input_data=[input_data, label])
 
 
+@tvm.testing.uses_gpu
+def test_forward_pool3d():
+    class Pool3D1(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs):
+            return nn.functional.avg_pool3d(inputs, kernel_size=2, stride=2, padding=0)
+
+    class Pool3D2(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs):
+            return nn.functional.adaptive_avg_pool3d(inputs, output_size=[3, 3, 3])
+
+    class Pool3D3(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs):
+            return nn.functional.avg_pool3d(
+                inputs,
+                kernel_size=3,
+                stride=1,
+                padding=[1, 1, 1],
+                exclusive=False,
+                divisor_override=2.5,
+            )
+
+    class Pool3D4(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs):
+            return nn.functional.avg_pool3d(
+                inputs,
+                kernel_size=2,
+                stride=1,
+                padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]],
+                ceil_mode=True,
+                data_format="NCDHW",
+            )
+
+    class Pool3D5(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs):
+            return nn.functional.avg_pool3d(
+                inputs,
+                kernel_size=2,
+                stride=1,
+                padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]],
+                ceil_mode=True,
+                data_format="NDHWC",
+            )
+
+    input_shapes = [[1, 2, 2, 8, 8], [1, 2, 3, 10, 10]]  # [N, C, D, H, W]
+    for input_shape in input_shapes:
+        input_data = paddle.uniform(shape=input_shape, dtype="float32", min=-1, max=1)
+        verify_model(Pool3D1(), input_data=input_data)
+        verify_model(Pool3D2(), input_data=input_data)
+        verify_model(Pool3D3(), input_data=input_data)
+        verify_model(Pool3D4(), input_data=input_data)
+        verify_model(Pool3D5(), input_data=input_data)
+
+
+@tvm.testing.uses_gpu
+def test_forward_set_value():
+    class SetValue(nn.Layer):
+        @paddle.jit.to_static
+        def forward(self, inputs, update_input):
+            x = inputs + 1
+            x[3:] = 3
+            x[1:] = 3.0
+            x[2:] = update_input
+            x[0] = 1
+            x[-3:-2] = 1
+            x[0][0] = 5
+            return x
+
+    input_shapes = [[5, 2], [10, 3], [10, 3, 3]]
+    for input_shape in input_shapes:
+        input_data = paddle.uniform(shape=input_shape, dtype="float32", min=-1, max=1)
+        update_shape = input_shape.copy()
+        update_shape[0] = input_shape[0] - 2
+        update_input = paddle.uniform(shape=update_shape, dtype="float32", min=-1, max=1)
+        verify_model(SetValue(), input_data=[input_data, update_input])
+
+
 if __name__ == "__main__":
     tvm.testing.main()

From d7b1f90fd5971f986af7f18b5ae1449c8d0b9a6d Mon Sep 17 00:00:00 2001
From: PuQing <me@puqing.work>
Date: Wed, 31 May 2023 19:28:31 +0800
Subject: [PATCH 6/6] reset gelu

---
 python/tvm/relay/frontend/paddlepaddle.py     | 26 +++----------------
 .../frontend/paddlepaddle/test_forward.py     | 13 +++-------
 2 files changed, 8 insertions(+), 31 deletions(-)

diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py
index c35330329bbe..8c06e14efa36 100755
--- a/python/tvm/relay/frontend/paddlepaddle.py
+++ b/python/tvm/relay/frontend/paddlepaddle.py
@@ -829,28 +829,10 @@ def convert_gelu(g, op, block):
     """Operator converter for gelu."""
 
     x = g.get_node(op.input("X")[0])
-    approx = op.attr("approximate")
-    if approx:
-        M_2_SQRTPI = 1.12837916709551257390
-        M_SQRT1_2 = 0.70710678118654752440
-        out = x * (
-            _expr.const(0.5, dtype="float32")
-            + _op.tanh(
-                x
-                * _expr.const(M_2_SQRTPI * M_SQRT1_2, dtype="float32")
-                * (
-                    _expr.const(1.0, dtype="float32")
-                    + _expr.const((0.044715), dtype="float32") * x * x
-                )
-            )
-            * _expr.const(0.5, dtype="float32")
-        )
-    else:
-        out = x * (
-            _expr.const(0.5, dtype="float32")
-            + _op.erf(x * _expr.const(0.5**0.5, dtype="float32"))
-            * _expr.const(0.5, dtype="float32")
-        )
+    out = x * (
+        _expr.const(0.5, dtype="float32")
+        + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32")
+    )
     g.add_node(op.output("Out")[0], out)
 
 
diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py
index 2b1b2a1cc75a..e5624530c5b9 100755
--- a/tests/python/frontend/paddlepaddle/test_forward.py
+++ b/tests/python/frontend/paddlepaddle/test_forward.py
@@ -1005,18 +1005,13 @@ def ones_like2(inputs):
 
 @tvm.testing.uses_gpu
 def test_forward_gelu():
-    class Gelu(nn.Layer):
-        def __init__(self, approximate=False):
-            super(Gelu, self).__init__()
-            self.approximate = approximate
-
-        def forward(self, inputs):
-            return nn.functional.gelu(inputs, approximate=self.approximate)
+    @paddle.jit.to_static
+    def gelu(inputs):
+        return nn.functional.gelu(inputs)
 
     input_shape = [1, 3, 10, 10]
     input_data = paddle.rand(input_shape, dtype="float32")
-    verify_model(Gelu(), input_data=input_data)
-    verify_model(Gelu(approximate=True), input_data=input_data)
+    verify_model(gelu, input_data=input_data)
 
 
 @tvm.testing.uses_gpu