diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c389cf92913e..5db09f322b0e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1254,11 +1254,19 @@ integrationtest_ubuntu_cpu_onnx() { export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py pytest $COV_ARG --verbose tests/python-pytest/onnx/mxnet_export_test.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest -n 24 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_bert_inference_onnxruntime + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[mobilenetv3_large] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnest200] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[resnet50_v2] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_class_model_inference_onnxruntime[vgg19_bn] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_obj_detection_model_inference_onnxruntime[center_net_resnet101_v1b_voc] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_img_segmentation_model_inference_onnxruntime[deeplab_resnet50_citys] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_pose_estimation_model_inference_onnxruntime[mobile_pose_mobilenet1.0] + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py::test_action_recognition_model_inference_onnxruntime[inceptionv3_kinetics400] } integrationtest_ubuntu_gpu_python() { @@ -1587,6 +1595,15 @@ nightly_estimator() { nosetests test_sentiment_rnn.py } +nightly_onnx_tests() { + set -ex + export PYTHONPATH=./python/ + export MXNET_SUBGRAPH_VERBOSE=0 + export DMLC_LOG_STACK_TRACE_DEPTH=10 + COV_ARG="--cov=./ --cov-report=xml --cov-append" + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py +} + # For testing PRs deploy_docs() { set -ex diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index a40e66a407d5..c985a182bb28 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1184,7 +1184,7 @@ def test_unix_julia10_cpu() { } def test_unix_onnx_cpu() { - return ['Onnx CPU': { + return ['Onnx CPU - Unit Tests': { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { diff --git a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py index 778678e6b5d4..a2d9a6bf1c2c 100644 --- a/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py +++ b/python/mxnet/contrib/onnx/mx2onnx/_op_translations.py @@ -787,10 +787,10 @@ def convert_copy(node, **kwargs): @mx_op.register("identity") def convert_identity(node, **kwargs): - """Map MXNet's identity operator attributes to onnx's ConstantFill operator + """Map MXNet's identity operator attributes to onnx's Identity operator and return the created node. """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("InstanceNorm") def convert_instancenorm(node, **kwargs): @@ -1017,12 +1017,12 @@ def convert_logistic_regression_output(node, **kwargs): @mx_op.register("BlockGrad") def convert_blockgrad(node, **kwargs): """ Skip operator """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("MakeLoss") def convert_makeloss(node, **kwargs): """ Skip operator """ - return create_basic_op_node('ConstantFill', node, kwargs) + return create_basic_op_node('Identity', node, kwargs) @mx_op.register("Concat") def convert_concat(node, **kwargs): diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 1a7ef1aaaa05..00f543fb58cb 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -93,6 +93,14 @@ core_logic: { utils.docker_run('ubuntu_nightly_gpu', 'nightly_estimator', true) } } + }, + 'ONNX: CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/onnx-test-cpu') { + utils.unpack_and_init('cpu_int64', mx_cmake_lib) + utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_tests', false) + } + } } } } diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 60e84d047439..ce372c78e250 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -59,15 +59,6 @@ def predict(self, data): return self.model(data) -def download_test_images(image_urls, tmpdir): - from urllib.parse import urlparse - paths = [] - for url in image_urls: - filename = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) - mx.test_utils.download(url, fname=filename) - paths.append(filename) - return paths - @with_seed() @pytest.mark.parametrize('model', ['bert_12_768_12']) @@ -123,6 +114,30 @@ def test_bert_inference_onnxruntime(tmp_path, model): shutil.rmtree(tmp_path) + +@pytest.fixture(scope="session") +def obj_class_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obj_class_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths + @pytest.mark.parametrize('model', [ 'alexnet', 'cifar_resnet20_v1', @@ -151,13 +166,12 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'mobilenetv2_0.25', 'mobilenetv3_large', 'mobilenetv3_small', - # failing due to accuracy - #'resnest14', - #'resnest26', - #'resnest50', - #'resnest101', - #'resnest200', - #'resnest269', + 'resnest14', + 'resnest26', + 'resnest50', + 'resnest101', + 'resnest200', + 'resnest269', 'resnet18_v1', 'resnet18_v1b_0.89', 'resnet18_v2', @@ -191,23 +205,26 @@ def test_bert_inference_onnxruntime(tmp_path, model): 'vgg16', 'vgg16_bn', 'vgg19', - 'vgg19_bn' + 'vgg19_bn', + 'xception', + 'inceptionv3' ]) -def test_obj_class_model_inference_onnxruntime(tmp_path, model): +def test_obj_class_model_inference_onnxruntime(tmp_path, model, obj_class_test_images): + inlen = 299 if 'inceptionv3' == model else 224 def normalize_image(imgfile): img_data = mx.image.imread(imgfile) - img_data = mx.image.imresize(img_data, 224, 224) + img_data = mx.image.imresize(img_data, inlen, inlen) img_data = img_data.transpose([2, 0, 1]).astype('float32') mean_vec = mx.nd.array([0.485, 0.456, 0.406]) stddev_vec = mx.nd.array([0.229, 0.224, 0.225]) norm_img_data = mx.nd.zeros(img_data.shape).astype('float32') for i in range(img_data.shape[0]): norm_img_data[i,:,:] = (img_data[i,:,:]/255 - mean_vec[i]) / stddev_vec[i] - return norm_img_data.reshape(1, 3, 224, 224).astype('float32') + return norm_img_data.reshape(1, 3, inlen, inlen).astype('float32') try: tmp_path = str(tmp_path) - M = GluonModel(model, (1,3,224,224), 'float32', tmp_path) + M = GluonModel(model, (1,3,inlen,inlen), 'float32', tmp_path) onnx_file = M.export_onnx() # create onnxruntime session using the generated onnx file @@ -216,20 +233,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in obj_class_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()})[0] @@ -239,6 +243,26 @@ def normalize_image(imgfile): shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def obj_detection_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("obj_det_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ @@ -246,10 +270,27 @@ def normalize_image(imgfile): 'center_net_resnet50_v1b_voc', 'center_net_resnet101_v1b_voc', 'center_net_resnet18_v1b_coco', - #'center_net_resnet50_v1b_coco', - 'center_net_resnet101_v1b_coco' + 'center_net_resnet50_v1b_coco', + 'center_net_resnet101_v1b_coco', + # the following models are failing due to onnxruntime errors + #'ssd_300_vgg16_atrous_voc', + #'ssd_512_vgg16_atrous_voc', + #'ssd_512_resnet50_v1_voc', + #'ssd_512_mobilenet1.0_voc', + #'faster_rcnn_resnet50_v1b_voc', + #'yolo3_darknet53_voc', + #'yolo3_mobilenet1.0_voc', + #'ssd_300_vgg16_atrous_coco', + #'ssd_512_vgg16_atrous_coco', + #'ssd_300_resnet34_v1b_coco', + #'ssd_512_resnet50_v1_coco', + #'ssd_512_mobilenet1.0_coco', + #'faster_rcnn_resnet50_v1b_coco', + #'faster_rcnn_resnet101_v1d_coco', + #'yolo3_darknet53_coco', + #'yolo3_mobilenet1.0_coco', ]) -def test_obj_detection_model_inference_onnxruntime(tmp_path, model): +def test_obj_detection_model_inference_onnxruntime(tmp_path, model, obj_detection_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile) img, _ = mx.image.center_crop(img, size=(512, 512)) @@ -266,18 +307,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in obj_detection_test_images: img_data = normalize_image(img) mx_class_ids, mx_scores, mx_boxes = M.predict(img_data) onnx_scores, onnx_class_ids, onnx_boxes = session.run([], {input_name: img_data.asnumpy()}) @@ -288,17 +318,38 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def img_segmentation_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("img_seg_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ 'fcn_resnet50_ade', 'fcn_resnet101_ade', 'deeplab_resnet50_ade', 'deeplab_resnet101_ade', - # the 4 models below are failing due to an accuracy issue - #'deeplab_resnest50_ade', - #'deeplab_resnest101_ade', - #'deeplab_resnest200_ade', - #'deeplab_resnest269_ade', + 'deeplab_resnest50_ade', + 'deeplab_resnest101_ade', + 'deeplab_resnest200_ade', + 'deeplab_resnest269_ade', 'fcn_resnet101_coco', 'deeplab_resnet101_coco', 'fcn_resnet101_voc', @@ -308,7 +359,7 @@ def normalize_image(imgfile): 'deeplab_resnet101_citys', 'deeplab_v3b_plus_wideresnet_citys' ]) -def test_img_segmentation_model_inference_onnxruntime(tmp_path, model): +def test_img_segmentation_model_inference_onnxruntime(tmp_path, model, img_segmentation_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile).astype('float32') img, _ = mx.image.center_crop(img, size=(480, 480)) @@ -326,20 +377,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in img_segmentation_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) @@ -350,6 +388,25 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) + +@pytest.fixture(scope="session") +def pose_estimation_test_images(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("pose_est_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths + @pytest.mark.parametrize('model', [ 'simple_pose_resnet18_v1b', 'simple_pose_resnet50_v1b', @@ -358,15 +415,15 @@ def normalize_image(imgfile): 'simple_pose_resnet101_v1d', 'simple_pose_resnet152_v1b', 'simple_pose_resnet152_v1d', - #'mobile_pose_resnet18_v1b', - #'mobile_pose_resnet50_v1b', - #'mobile_pose_mobilenet1.0', - #'mobile_pose_mobilenetv2_1.0', - #'mobile_pose_mobilenetv3_large', - #'mobile_pose_mobilenetv3_small', - #'alpha_pose_resnet101_v1b_coco', + 'alpha_pose_resnet101_v1b_coco', + 'mobile_pose_resnet18_v1b', + 'mobile_pose_resnet50_v1b', + 'mobile_pose_mobilenet1.0', + 'mobile_pose_mobilenetv2_1.0', + 'mobile_pose_mobilenetv3_large', + 'mobile_pose_mobilenetv3_small', ]) -def test_pose_estimation_model_inference_onnxruntime(tmp_path, model): +def test_pose_estimation_model_inference_onnxruntime(tmp_path, model, pose_estimation_test_images): def normalize_image(imgfile): img = mx.image.imread(imgfile).astype('float32') img, _ = mx.image.center_crop(img, size=(512, 512)) @@ -383,15 +440,7 @@ def normalize_image(imgfile): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_image_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', - ] - - for img in download_test_images(test_image_urls, tmp_path): + for img in pose_estimation_test_images: img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) @@ -402,22 +451,37 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) +@pytest.fixture(scope="session") +def act_recognition_test_data(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("act_rec_data") + from urllib.parse import urlparse + test_image_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/biking.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/diving.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/golfing.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/sledding.rec', + ] + paths = [] + for url in test_image_urls: + fn = os.path.join(tmpdir, os.path.basename(urlparse(url).path)) + mx.test_utils.download(url, fname=fn) + paths.append(fn) + return paths @pytest.mark.parametrize('model', [ 'inceptionv1_kinetics400', 'resnet18_v1b_kinetics400', 'resnet34_v1b_kinetics400', + 'resnet50_v1b_kinetics400', + 'resnet101_v1b_kinetics400', + 'resnet152_v1b_kinetics400', 'resnet50_v1b_hmdb51', 'resnet50_v1b_sthsthv2', 'vgg16_ucf101', - # the following models are failing due to an accuracy issue - #'resnet50_v1b_kinetics400', - #'resnet101_v1b_kinetics400', - #'resnet152_v1b_kinetics400', - #'inceptionv3_kinetics400', - #'inceptionv3_ucf101', + 'inceptionv3_kinetics400', + 'inceptionv3_ucf101', ]) -def test_action_recognition_model_inference_onnxruntime(tmp_path, model): +def test_action_recognition_model_inference_onnxruntime(tmp_path, model, act_recognition_test_data): batch_size = 64 input_len = 224 if 'inceptionv3' in model: @@ -438,18 +502,11 @@ def load_video(filepath): session = onnxruntime.InferenceSession(onnx_file, ses_opt) input_name = session.get_inputs()[0].name - test_video_urls = [ - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/biking.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/diving.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/golfing.rec', - 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/sledding.rec', - ] - - for video in download_test_images(test_video_urls, tmp_path): + for video in act_recognition_test_data: data = load_video(video) mx_result = M.predict(data) onnx_result = session.run([], {input_name: data.asnumpy()})[0] - assert_almost_equal(mx_result, onnx_result) + assert_almost_equal(mx_result, onnx_result, rtol=0.001, atol=0.01) finally: shutil.rmtree(tmp_path)