[v1.x] Rearrange ONNX tests in Nightly CI

ci/docker/runtime_functions.sh

@@ -1263,18 +1263,8 @@ integrationtest_ubuntu_cpu_onnx() {
     # Skip this as https://github.com/apache/incubator-mxnet/pull/19914 breaks import
     #pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py
     #pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[mobilenetv3_large]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnest200]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnet50_v2]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[vgg19_bn]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime[center_net_resnet101_v1b_voc]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime[deeplab_resnet50_citys]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime[mobile_pose_mobilenet1.0]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime[inceptionv3_kinetics400]
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_dynamic_shape_bert_inference_onnxruntime
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_dynamic_shape_cv_inference_onnxruntime
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_transformer_pretrained_inference_onnxruntime
+    pytest $COV_ARG -v -m integration tests/python-pytest/onnx/test_onnxruntime_cv.py
+    pytest $COV_ARG -v -m integration tests/python-pytest/onnx/test_onnxruntime_nlp.py
 }
 
 integrationtest_ubuntu_gpu_python() {
@@ -1604,13 +1594,22 @@ nightly_estimator() {
     nosetests test_sentiment_rnn.py
 }
 
-nightly_onnx_tests() {
+nightly_onnx_cv_tests() {
     set -ex
     export PYTHONPATH=./python/
     export MXNET_SUBGRAPH_VERBOSE=0
     export DMLC_LOG_STACK_TRACE_DEPTH=10
     COV_ARG="--cov=./ --cov-report=xml --cov-append"
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime_cv.py
+}
+
+nightly_onnx_nlp_tests() {
+    set -ex
+    export PYTHONPATH=./python/
+    export MXNET_SUBGRAPH_VERBOSE=0
+    export DMLC_LOG_STACK_TRACE_DEPTH=10
+    COV_ARG="--cov=./ --cov-report=xml --cov-append"
+    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime_nlp.py
 }
 
 # For testing PRs

pytest.ini

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[pytest]
+markers =
+    seed: set the python, numpy and mxnet random seeds to a specified value for test reproducibility
+    serial: mark a test that requires more resources to run that are thus only suitable for serial run.
+    remote_required: mark a test that requires internet access.
+    gpu: mark a test that requires GPU.
+    integration: mark an integration test
+    onnx_coverage: ONNX coverage test
+    garbage_expected: this test leaks ndarray references. The tested functionality is broken or there is a Python bug.
+
+env =
+    MXNET_HOME=tests/data
+
+timeout = 1200
+faulthandler_timeout = 1200

tests/nightly/JenkinsfileForBinaries

@@ -94,11 +94,19 @@ core_logic: {
         }
       }
     },
-    'ONNX: CPU': {
+    'ONNX-CV: CPU': {
       node(NODE_LINUX_CPU) {
-        ws('workspace/onnx-test-cpu') {
+        ws('workspace/onnx-cv-test-cpu') {
           utils.unpack_and_init('cpu_int64', mx_cmake_lib)
-          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_tests', false)
+          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_cv_tests', false)
+        }
+      }
+    }
+    'ONNX-NLP: CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/onnx-nlp-test-cpu') {
+          utils.unpack_and_init('cpu_int64', mx_cmake_lib)
+          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_nlp_tests', false)
         }
       }
     }