From f0826a51296a331052f19e5c9b4b64bbbbc98bdf Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Tue, 9 Mar 2021 00:47:22 +0000 Subject: [PATCH 1/2] add roberta tests --- tests/python-pytest/onnx/test_onnxruntime.py | 61 +++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 48a8a436384b..5cb4fe7d89c4 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -555,8 +555,67 @@ def load_video(filepath): finally: shutil.rmtree(tmp_path) + @with_seed() -@pytest.mark.parametrize('model', ['bert_12_768_12']) +@pytest.mark.parametrize('model_name', ['roberta_24_1024_16', 'roberta_12_768_12']) +def test_roberta_inference_onnxruntime(tmp_path, model_name): + tmp_path = str(tmp_path) + try: + import gluonnlp as nlp + ctx = mx.cpu(0) + + dataset= 'openwebtext_ccnews_stories_books_cased'#'book_corpus_wiki_en_uncased' + model, _ = nlp.model.get_model( + name=model_name, + ctx=ctx, + pretrained=True, + use_decoder=True, + dataset_name=dataset) + + model.hybridize(static_alloc=False) + + batch = 2 + seq_length = 32 + num_masked_positions = 1 + inputs = mx.nd.random.uniform(0, 30522, shape=(batch, seq_length), dtype='float32', ctx=ctx) + valid_length = mx.nd.array([seq_length] * batch, dtype='float32', ctx=ctx) + masked_positions = mx.nd.random.uniform(0, 32, shape=(batch, num_masked_positions), + dtype='float32', ctx=ctx).astype('int32') + + sequence_outputs, attention_outputs= model(inputs, valid_length, masked_positions) + + model_dir = f'roberta_model' + if not os.path.isdir(model_dir): + os.mkdir(model_dir) + + prefix = '%s/%s' % (model_dir, model_name) + model.export(prefix) + + sym_file = "%s-symbol.json" % prefix + params_file = "%s-0000.params" % prefix + onnx_file = "%s.onnx" % prefix + input_shapes = [(batch, seq_length), (batch,), (batch, num_masked_positions)] + converted_model_path = mx.contrib.onnx.export_model(sym_file, params_file, input_shapes, + [np.float32, np.float32, np.int32], + onnx_file, verbose=True) + + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + sess = onnxruntime.InferenceSession(onnx_file, sess_options) + + in_tensors = [inputs, valid_length, masked_positions] + input_dict = dict((sess.get_inputs()[i].name, in_tensors[i].asnumpy()) for i in range(len(in_tensors))) + pred = sess.run(None, input_dict) + + assert_almost_equal(sequence_outputs, pred[0]) + assert_almost_equal(attension_outputs, pred[1]) + + finally: + shutil.rmtree(tmp_path) + + +@with_seed() +@pytest.mark.parametrize('model', ['bert_12_768_12', 'bert_24_1024_16']) def test_bert_inference_onnxruntime(tmp_path, model): tmp_path = str(tmp_path) try: From 38c0456780bd5c7730f5666585ed64e451529ad4 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Tue, 9 Mar 2021 18:47:02 +0000 Subject: [PATCH 2/2] add distil bert test --- tests/python-pytest/onnx/test_onnxruntime.py | 55 ++++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/tests/python-pytest/onnx/test_onnxruntime.py b/tests/python-pytest/onnx/test_onnxruntime.py index 5cb4fe7d89c4..3bcf27f7f1c9 100644 --- a/tests/python-pytest/onnx/test_onnxruntime.py +++ b/tests/python-pytest/onnx/test_onnxruntime.py @@ -596,8 +596,8 @@ def test_roberta_inference_onnxruntime(tmp_path, model_name): onnx_file = "%s.onnx" % prefix input_shapes = [(batch, seq_length), (batch,), (batch, num_masked_positions)] converted_model_path = mx.contrib.onnx.export_model(sym_file, params_file, input_shapes, - [np.float32, np.float32, np.int32], - onnx_file, verbose=True) + [np.float32, np.float32, np.int32], + onnx_file, verbose=True) sess_options = onnxruntime.SessionOptions() sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL @@ -626,11 +626,11 @@ def test_bert_inference_onnxruntime(tmp_path, model): name=model, ctx=ctx, dataset_name=dataset, - pretrained=False, + pretrained=True, use_pooler=True, use_decoder=False, use_classifier=False) - model.initialize(ctx=ctx) + model.hybridize(static_alloc=True) batch = 5 @@ -669,3 +669,50 @@ def test_bert_inference_onnxruntime(tmp_path, model): shutil.rmtree(tmp_path) +@with_seed() +@pytest.mark.parametrize('model_name', ['distilbert_6_768_12']) +def test_distilbert_inference_onnxruntime(tmp_path, model_name): + tmp_path = str(tmp_path) + try: + import gluonnlp as nlp + dataset = 'distilbert_book_corpus_wiki_en_uncased' + ctx = mx.cpu(0) + model, _ = nlp.model.get_model( + name=model_name, + ctx=ctx, + pretrained=True, + dataset_name=dataset) + + model.hybridize(static_alloc=True) + + batch = 2 + seq_length = 32 + num_masked_positions = 1 + inputs = mx.nd.random.uniform(0, 30522, shape=(batch, seq_length), dtype='float32', ctx=ctx) + valid_length = mx.nd.array([seq_length] * batch, dtype='float32', ctx=ctx) + + sequence_outputs = model(inputs, valid_length) + + prefix = "%s/distilbert" % tmp_path + model.export(prefix) + sym_file = "%s-symbol.json" % prefix + params_file = "%s-0000.params" % prefix + onnx_file = "%s.onnx" % prefix + + input_shapes = [(batch, seq_length), (batch,)] + converted_model_path = mx.contrib.onnx.export_model(sym_file, params_file, input_shapes, + [np.float32, np.float32], + onnx_file, verbose=True) + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + sess = onnxruntime.InferenceSession(onnx_file, sess_options) + + in_tensors = [inputs, valid_length] + input_dict = dict((sess.get_inputs()[i].name, in_tensors[i].asnumpy()) for i in range(len(in_tensors))) + pred = sess.run(None, input_dict) + + assert_almost_equal(sequence_outputs, pred[0]) + + finally: + shutil.rmtree(tmp_path) +