From 605917493f2bae0323f2d55406616298401af99f Mon Sep 17 00:00:00 2001 From: Dmitriy Smirnov Date: Mon, 14 Dec 2020 15:08:23 +0000 Subject: [PATCH 1/4] [TFLite] Added check for dynamic range quantization Added check to prevent optimized with "dynamic range quantization" tflite files to be loaded as the optimization is not fully supported. https://www.tensorflow.org/lite/performance/post_training_quantization#dynamic_range_quantization --- python/tvm/relay/frontend/tflite.py | 27 +++++++++++++++++--- tests/python/frontend/tflite/test_forward.py | 21 +++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/python/tvm/relay/frontend/tflite.py b/python/tvm/relay/frontend/tflite.py index 54eeb9d82447..ad938b0d5aab 100644 --- a/python/tvm/relay/frontend/tflite.py +++ b/python/tvm/relay/frontend/tflite.py @@ -174,17 +174,38 @@ def __init__(self, model, subgraph, exp_tab): def check_unsupported_ops(self): """Check unsupported TFLite ops in our converter.""" unsupported_ops_set = set() - + dynamic_range_ops_set = set() for op_idx in range(self.subgraph.OperatorsLength()): op = self.subgraph.Operators(op_idx) op_code_str = self.get_op_code_str(op) if op_code_str not in self.convert_map: unsupported_ops_set.add(op_code_str) + continue + + # Trying to exclude "dynamic range quantization" optimized ops as not supported in TVM + qnn_in_cnt = len( + [_.qnn_params for _ in self.get_input_tensors(op)[1:] if _.qnn_params is not None] + ) + qnn_out_cnt = len( + [_.qnn_params for _ in self.get_output_tensors(op) if _.qnn_params is not None] + ) + + if qnn_out_cnt == 0 and qnn_in_cnt > 0: + dynamic_range_ops_set.add(op_code_str) + + raise_msg = "" if unsupported_ops_set: - msg = "The following operators are not supported in frontend " "TFLite: {}" + msg = "The following operators are not supported in frontend " "TFLite: {}\n" ops = str(list(unsupported_ops_set)).strip("[,]") - raise tvm.error.OpNotImplemented(msg.format(ops)) + raise_msg += msg.format(ops) + + if dynamic_range_ops_set: + msg = "The following operators are likely to have dynamic range quantization: {}. If you are running an optimized graph, please turn off dynamic range quantization or use full integer quantization" + raise_msg += msg.format(str(list(dynamic_range_ops_set)).strip("[,]")) + + if len(raise_msg) > 0: + raise tvm.error.OpNotImplemented(raise_msg) def convert_op_to_relay(self): """Convert TFLite ops to relay ops""" diff --git a/tests/python/frontend/tflite/test_forward.py b/tests/python/frontend/tflite/test_forward.py index 56c50c315a70..be14ff3c39c4 100644 --- a/tests/python/frontend/tflite/test_forward.py +++ b/tests/python/frontend/tflite/test_forward.py @@ -3941,6 +3941,27 @@ def test_forward_mediapipe_hand_landmark(): ) +####################################################################### +# Test check for Tensorflow "dynamic range quantization" optimization +# -------------- +def test_prevent_tensorflow_dynamic_range(): + """ + Should prevent runnung "dynamic range quantization" optimized TFLite graph + """ + data_array = np.random.randint(0, 2, (1, 1024, 1024)).astype(dtype=np.float32) + filter_array = np.random.randint(0, 2, (1024, 1024)).astype(dtype=np.float32) + data_in = tf.keras.layers.Input(shape=data_array.shape[1:]) + dense = tf.keras.layers.Dense(units=filter_array.shape[-1], use_bias=False)(data_in) + keras_model = tf.keras.models.Model(data_in, dense) + keras_model.layers[1].set_weights([filter_array]) + + converter = interpreter_wrapper.TFLiteConverter.from_keras_model(keras_model) + converter.optimizations = [tf.lite.Optimize.DEFAULT] + tflite_model = converter.convert() + with pytest.raises(tvm.error.OpNotImplemented): + tvm_output = run_tvm_graph(tflite_model, data_array, data_in.name.replace(":0", "")) + + ####################################################################### # Main # ---- From 19be0d9f0558701150cbecd1be21c04064d42763 Mon Sep 17 00:00:00 2001 From: Dmitriy Smirnov Date: Tue, 15 Dec 2020 19:24:36 +0000 Subject: [PATCH 2/4] linter --- tests/python/frontend/tflite/test_forward.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/frontend/tflite/test_forward.py b/tests/python/frontend/tflite/test_forward.py index be14ff3c39c4..1f61a055e210 100644 --- a/tests/python/frontend/tflite/test_forward.py +++ b/tests/python/frontend/tflite/test_forward.py @@ -3946,7 +3946,7 @@ def test_forward_mediapipe_hand_landmark(): # -------------- def test_prevent_tensorflow_dynamic_range(): """ - Should prevent runnung "dynamic range quantization" optimized TFLite graph + Should prevent runnung "dynamic range quantization" optimized TFLite graph """ data_array = np.random.randint(0, 2, (1, 1024, 1024)).astype(dtype=np.float32) filter_array = np.random.randint(0, 2, (1024, 1024)).astype(dtype=np.float32) From 77b39ab3a39d5cf9678aac50b751aa40c908a7a5 Mon Sep 17 00:00:00 2001 From: Dmitriy Smirnov Date: Tue, 15 Dec 2020 19:42:59 +0000 Subject: [PATCH 3/4] linter --- python/tvm/relay/frontend/tflite.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/tvm/relay/frontend/tflite.py b/python/tvm/relay/frontend/tflite.py index ad938b0d5aab..ddd2e6dd1174 100644 --- a/python/tvm/relay/frontend/tflite.py +++ b/python/tvm/relay/frontend/tflite.py @@ -201,7 +201,11 @@ def check_unsupported_ops(self): raise_msg += msg.format(ops) if dynamic_range_ops_set: - msg = "The following operators are likely to have dynamic range quantization: {}. If you are running an optimized graph, please turn off dynamic range quantization or use full integer quantization" + msg = ( + "The following operators are likely to have dynamic range quantization: {}. " + "If you are running an optimized graph, please turn off dynamic range quantization " + "or use full integer quantization" + ) raise_msg += msg.format(str(list(dynamic_range_ops_set)).strip("[,]")) if len(raise_msg) > 0: From 678ff420c99f8b223ae80891b5c1d0694e2dcb80 Mon Sep 17 00:00:00 2001 From: Dmitriy Smirnov Date: Wed, 16 Dec 2020 13:21:01 +0000 Subject: [PATCH 4/4] unit test fix --- python/tvm/relay/frontend/tflite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/tvm/relay/frontend/tflite.py b/python/tvm/relay/frontend/tflite.py index ddd2e6dd1174..a010555938f9 100644 --- a/python/tvm/relay/frontend/tflite.py +++ b/python/tvm/relay/frontend/tflite.py @@ -184,13 +184,16 @@ def check_unsupported_ops(self): # Trying to exclude "dynamic range quantization" optimized ops as not supported in TVM qnn_in_cnt = len( + [_.qnn_params for _ in self.get_input_tensors(op)[0:1] if _.qnn_params is not None] + ) + qnn_weight_cnt = len( [_.qnn_params for _ in self.get_input_tensors(op)[1:] if _.qnn_params is not None] ) qnn_out_cnt = len( [_.qnn_params for _ in self.get_output_tensors(op) if _.qnn_params is not None] ) - if qnn_out_cnt == 0 and qnn_in_cnt > 0: + if qnn_in_cnt == 0 and qnn_out_cnt == 0 and qnn_weight_cnt > 0: dynamic_range_ops_set.add(op_code_str) raise_msg = ""