From 990952bb567b70c1e80c7c64c5d58070c1504089 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Feb 2021 03:43:47 +0000 Subject: [PATCH 1/8] Add pose estimation model end-to-end tests for supported models. --- tests/python-pytest/onnx/test_onnxruntime.py | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 34b6ff0367fa..e94cf3858339 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -286,6 +286,56 @@ def normalize_image(imgfile): finally: shutil.rmtree(tmp_path) +@pytest.mark.parametrize('model', [ + 'simple_pose_resnet18_v1b', + 'simple_pose_resnet50_v1b', + 'simple_pose_resnet50_v1d', + 'simple_pose_resnet101_v1b', + 'simple_pose_resnet101_v1d', + 'simple_pose_resnet152_v1b', + 'simple_pose_resnet152_v1d', + #'mobile_pose_resnet18_v1b', + #'mobile_pose_resnet50_v1b', + #'mobile_pose_mobilenet1.0', + #'mobile_pose_mobilenetv2_1.0', + #'mobile_pose_mobilenetv3_large', + #'mobile_pose_mobilenetv3_small', + #'alpha_pose_resnet101_v1b_coco', +]) +def test_pose_estimation_model_inference_onnxruntime(tmp_path, model): + def normalize_image(imgfile): + img = mx.image.imread(imgfile).astype('float32') + img, _ = mx.image.center_crop(img, size=(512, 512)) + img = gluoncv.data.transforms.presets.segmentation.test_transform(img, mx.cpu(0)) + return img + + try: + tmp_path = str(tmp_path) + M = GluonModel(model, (1,3,512,512), 'float32', tmp_path) + onnx_file = M.export_onnx() + # create onnxruntime session using the generated onnx file + ses_opt = onnxruntime.SessionOptions() + ses_opt.log_severity_level = 3 + session = onnxruntime.InferenceSession(onnx_file, ses_opt) + input_name = session.get_inputs()[0].name + + test_image_urls = [ + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/soccer.png', + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/tennis.jpg', + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/bikes.jpg', + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/runners.jpg', + ] + + for img in download_test_images(test_image_urls, tmp_path): + img_data = normalize_image(os.path.join(tmp_path, img)) + mx_result = M.predict(img_data) + onnx_result = session.run([], {input_name: img_data.asnumpy()}) + assert(len(mx_result) == len(onnx_result)) + for i in range(len(mx_result)): + assert_almost_equal(mx_result[i], onnx_result[i]) + + finally: + shutil.rmtree(tmp_path) @with_seed() From 50de40eda8d05080b1b0af5b7e8dc043eb7dfb1b Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Feb 2021 04:35:43 +0000 Subject: [PATCH 2/8] Run onnxruntime tests with 4 processes to cut down runtime. --- ci/docker/runtime_functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 0bbc6d79b9b8..d3b78b3e2ee9 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1278,13 +1278,12 @@ integrationtest_ubuntu_cpu_onnx() { export PYTHONPATH=./python/ export MXNET_SUBGRAPH_VERBOSE=0 export DMLC_LOG_STACK_TRACE_DEPTH=10 - #tests/python-pytest/onnx/backend_test.py COV_ARG="--cov=./ --cov-report=xml --cov-append" pytest $COV_ARG --verbose tests/python-pytest/onnx/mxnet_export_test.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py } integrationtest_ubuntu_gpu_python() { From 65c3aecbd2452352c6359a9cb0976e2531eccf08 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Feb 2021 18:02:22 +0000 Subject: [PATCH 3/8] Add new pose test image. --- tests/python-pytest/onnx/test_onnxruntime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index e94cf3858339..dfc869a5145f 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -321,9 +321,10 @@ def normalize_image(imgfile): test_image_urls = [ 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/soccer.png', + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/soccer2.jpg', 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/tennis.jpg', 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/bikes.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/runners.jpg', + 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/runners.jpg' ] for img in download_test_images(test_image_urls, tmp_path): From 76ff877eef7baa45b78adc88268e8ff85662f902 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Wed, 3 Feb 2021 22:32:07 +0000 Subject: [PATCH 4/8] Add action recognition end-to-end tests and update test image URLs. --- tests/python-pytest/onnx/test_onnxruntime.py | 122 ++++++++++++++----- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index dfc869a5145f..ad00bddf240f 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -140,7 +140,9 @@ def download_test_images(image_urls, tmpdir): ]) def test_obj_class_model_inference_onnxruntime(tmp_path, model): def normalize_image(imgfile): - img_data = mx.image.imread(imgfile).transpose([2, 0, 1]).astype('float32') + img_data = mx.image.imread(imgfile) + img_data = mx.image.imresize(img_data, 224, 224) + img_data = img_data.transpose([2, 0, 1]).astype('float32') mean_vec = mx.nd.array([0.485, 0.456, 0.406]) stddev_vec = mx.nd.array([0.229, 0.224, 0.225]) norm_img_data = mx.nd.zeros(img_data.shape).astype('float32') @@ -160,11 +162,16 @@ def normalize_image(imgfile): input_name = session.get_inputs()[0].name test_image_urls = [ - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/apron.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dolphin.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/hammerheadshark.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/lotus.jpg' + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', ] for img in download_test_images(test_image_urls, tmp_path): @@ -205,18 +212,18 @@ def normalize_image(imgfile): input_name = session.get_inputs()[0].name test_image_urls = [ - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog2.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog3.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/car6.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/apron.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dolphin.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/hammerheadshark.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/lotus.jpg' + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', ] for img in download_test_images(test_image_urls, tmp_path): - img_data = normalize_image(os.path.join(tmp_path, img)) + img_data = normalize_image(img) mx_class_ids, mx_scores, mx_boxes = M.predict(img_data) onnx_scores, onnx_class_ids, onnx_boxes = session.run([], {input_name: img_data.asnumpy()}) assert_almost_equal(mx_class_ids, onnx_class_ids) @@ -265,18 +272,20 @@ def normalize_image(imgfile): input_name = session.get_inputs()[0].name test_image_urls = [ - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog2.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog3.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/car6.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dog.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/apron.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/dolphin.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/hammerheadshark.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/lotus.jpg' + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/car.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/duck.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/flower.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/shark.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/tree.jpg', ] for img in download_test_images(test_image_urls, tmp_path): - img_data = normalize_image(os.path.join(tmp_path, img)) + img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) assert(len(mx_result) == len(onnx_result)) @@ -320,15 +329,15 @@ def normalize_image(imgfile): input_name = session.get_inputs()[0].name test_image_urls = [ - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/soccer.png', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/soccer2.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/tennis.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/bikes.jpg', - 'https://raw.githubusercontent.com/apache/incubator-mxnet-ci/master/test-data/images/runners.jpg' + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/bikers.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/dancer.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/fieldhockey.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/runners.jpg', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/images/soccer2.jpg', ] for img in download_test_images(test_image_urls, tmp_path): - img_data = normalize_image(os.path.join(tmp_path, img)) + img_data = normalize_image(img) mx_result = M.predict(img_data) onnx_result = session.run([], {input_name: img_data.asnumpy()}) assert(len(mx_result) == len(onnx_result)) @@ -339,6 +348,59 @@ def normalize_image(imgfile): shutil.rmtree(tmp_path) +@pytest.mark.parametrize('model', [ + 'inceptionv1_kinetics400', + 'resnet18_v1b_kinetics400', + 'resnet34_v1b_kinetics400', + 'resnet50_v1b_hmdb51', + 'resnet50_v1b_sthsthv2', + 'vgg16_ucf101', + # the following models are failing due to an accuracy issue + #'resnet50_v1b_kinetics400', + #'resnet101_v1b_kinetics400', + #'resnet152_v1b_kinetics400', + #'inceptionv3_kinetics400', + #'inceptionv3_ucf101', +]) +def test_action_recognition_model_inference_onnxruntime(tmp_path, model): + batch_size = 64 + input_len = 224 + if 'inceptionv3' in model: + input_len = 340 + + def load_video(filepath): + iterator = mx.image.ImageIter(batch_size=batch_size, data_shape=(3,input_len,input_len), path_imgrec=filepath) + for batch in iterator: + return batch.data[0] + + try: + tmp_path = str(tmp_path) + M = GluonModel(model, (batch_size,3,input_len,input_len), 'float32', tmp_path) + onnx_file = M.export_onnx() + # create onnxruntime session using the generated onnx file + ses_opt = onnxruntime.SessionOptions() + ses_opt.log_severity_level = 3 + session = onnxruntime.InferenceSession(onnx_file, ses_opt) + input_name = session.get_inputs()[0].name + + test_video_urls = [ + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/biking.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/diving.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/golfing.rec', + 'https://github.com/apache/incubator-mxnet-ci/raw/master/test-data/actions/sledding.rec', + ] + + for video in download_test_images(test_video_urls, tmp_path): + data = load_video(video) + mx_result = M.predict(data) + onnx_result = session.run([], {input_name: data.asnumpy()})[0] + assert_almost_equal(mx_result, onnx_result) + + finally: + shutil.rmtree(tmp_path) + + + @with_seed() @pytest.mark.parametrize('model', ['bert_12_768_12']) def test_bert_inference_onnxruntime(tmp_path, model): From f17b6b7602ea8935a38c0c210deb2b625d127cfd Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 4 Feb 2021 00:59:40 +0000 Subject: [PATCH 5/8] We can't run tests in parallel yet, need the required python module. --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d3b78b3e2ee9..a236c7b3f148 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1283,7 +1283,7 @@ integrationtest_ubuntu_cpu_onnx() { pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py + pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py } integrationtest_ubuntu_gpu_python() { From f8edb32478940533f637edef376294635d2cb45b Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 4 Feb 2021 01:16:54 +0000 Subject: [PATCH 6/8] Install pytest-xdist. --- ci/docker/install/ubuntu_onnx.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh index 6a5104a8910a..ecb9f4377687 100755 --- a/ci/docker/install/ubuntu_onnx.sh +++ b/ci/docker/install/ubuntu_onnx.sh @@ -30,5 +30,4 @@ echo "Installing libprotobuf-dev and protobuf-compiler ..." apt-get update || true apt-get install -y libprotobuf-dev protobuf-compiler -echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX, tabulate and onnxruntime..." -pip3 install pytest pytest-cov protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 gluonnlp gluoncv +pip3 install pytest pytest-cov pytest-xdist protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 gluonnlp gluoncv From a187cd05c0b2ab1078171c676d5e9f6cb797c59b Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 4 Feb 2021 01:17:33 +0000 Subject: [PATCH 7/8] Run tests in parallel with pytest-xdist installed. --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index a236c7b3f148..d3b78b3e2ee9 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1283,7 +1283,7 @@ integrationtest_ubuntu_cpu_onnx() { pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py + pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py } integrationtest_ubuntu_gpu_python() { From 3306bfc8aa3446d9973b2c9d5db34861256104ad Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 4 Feb 2021 05:07:32 +0000 Subject: [PATCH 8/8] Increase process count to 12 for onnxruntime tests. --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d3b78b3e2ee9..cf12dcb49707 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1283,7 +1283,7 @@ integrationtest_ubuntu_cpu_onnx() { pytest $COV_ARG --verbose tests/python-pytest/onnx/test_models.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_node.py pytest $COV_ARG --verbose tests/python-pytest/onnx/test_operators.py - pytest -n 4 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py + pytest -n 12 $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime.py } integrationtest_ubuntu_gpu_python() {