diff --git a/backends/nxp/backend/ir/converter/conversion/common.py b/backends/nxp/backend/ir/converter/conversion/common.py index 0f69b152ec7..8230e39a7fa 100755 --- a/backends/nxp/backend/ir/converter/conversion/common.py +++ b/backends/nxp/backend/ir/converter/conversion/common.py @@ -70,29 +70,22 @@ def try_get_input(t_op: tflite_model.Operator, idx: int) -> tflite_model.Tensor return tensor -def extend_1d_pads_to_2d(onnx_1d_pads: MutableSequence): - """Extend the onnx 'pads' operator attribute that represents padding for a 1D kernel to 2D, by adding '0's.""" - if onnx_1d_pads is not None: - onnx_1d_pads.insert(1, 0) - onnx_1d_pads.append(0) +def extend_1d_padding_to_2d(tflite_1d_padding: MutableSequence): + """Extend the PyTorch 'padding' operator attribute that represents padding for a 1D kernel to 2D, by adding '0's.""" + if tflite_1d_padding is not None: + tflite_1d_padding.append(0) -def extend_1d_strides_to_2d(onnx_1d_strides: MutableSequence): - """Extend the onnx 'strides' operator attribute that represents strides for a 1D kernel to 2D, by adding '1'.""" - if onnx_1d_strides is not None: - onnx_1d_strides.append(1) +def extend_1d_stride_to_2d(tflite_1d_stride: MutableSequence): + """Extend the PyTorch 'stride' operator attribute that represents stride for a 1D kernel to 2D, by adding '1'.""" + if tflite_1d_stride is not None: + tflite_1d_stride.append(1) -def extend_1d_dilations_to_2d(onnx_1d_dilations: MutableSequence): - """Extend the onnx 'dilations' operator attribute that represents dilations for a 1D kernel to 2D, by adding '1'.""" - if onnx_1d_dilations is not None: - onnx_1d_dilations.append(1) - - -def extend_1d_kernel_shape_to_2d(onnx_1d_kernel_shape: MutableSequence): - """Extend the onnx 1D 'kernel_shape' operator attribute to 2D, by adding '1'.""" - if onnx_1d_kernel_shape is not None: - onnx_1d_kernel_shape.append(1) +def extend_1d_dilation_to_2d(tflite_1d_dilation: MutableSequence): + """Extend the PyTorch 'dilation' operator attribute that represents dilation for a 1D kernel to 2D, by adding '1'.""" + if tflite_1d_dilation is not None: + tflite_1d_dilation.append(1) StridedOptions = ( diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py index db05f0e7ba3..49915311699 100644 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/convolution_converter.py @@ -14,8 +14,12 @@ from executorch.backends.nxp.backend.ir.converter.conversion import ( aten_translator, common, + translator, ) from executorch.backends.nxp.backend.ir.converter.conversion.common import try_get_input +from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( + tf_lite_type_to_numpy, +) from executorch.backends.nxp.backend.ir.converter.node_converter import ( NodeConverter, Target, @@ -36,6 +40,7 @@ from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import ( conv_2d_options, depthwise_conv_2d_options, + reshape_options, ) from torch.fx import Node from torch.nn import Parameter @@ -85,13 +90,15 @@ def _is_supported_on_target( def _is_supported_in_IR( node: Node, parameters_mapping: dict[str, Parameter] ) -> bool: + input_tensor_rank = len(node.meta["val"].shape) + dimensions = input_tensor_rank - 2 is_transposed = node.args[6] output_padding = node.args[7] if is_transposed: return False - if output_padding != [0, 0]: + if output_padding != [0] * dimensions: return False if input_tensor_safe(node, 2) is None: @@ -116,7 +123,107 @@ def _get_convolution_arguments( _, _, _, stride, padding, dilation, transposed, out_padding, groups = ( conv_node.args ) - return stride, padding, dilation, transposed, out_padding, groups + return ( + list(stride), + list(padding), + list(dilation), + transposed, + out_padding, + groups, + ) + + def _convert_1d_conv( + self, t_op: tflite_model.Operator, conv_params: ConvParameters + ) -> list[tflite_model.Operator]: + """Convert the 'Conv' operator with a 1D kernel to TFLite 'Conv2D'. + TFLite doesn't support 1D convolution, but this behaviour can be represented using + Reshape -> Conv2D -> Reshape. + The first reshape introduces a 4th dimension with size 1. The second Reshape removes the temporary dimension. + """ + # -- Calculate the shapes for equivalent 2D convolution -- + conv_2d_input_shape = translator.nhc_dimensions_to_nhwc( + t_op.tmp_inputs[0].shape.vector + ) + conv_2d_weight_shape = translator.nhc_dimensions_to_nhwc( + t_op.tmp_inputs[1].shape.vector + ) + conv_2d_output_shape = translator.nhc_dimensions_to_nhwc( + t_op.tmp_outputs[0].shape.vector + ) + + # -- Generate tensors taking part in the conversion -- + reshape1_input = t_op.tmp_inputs[0] + + reshape1_output = self.builder.duplicate_tensor( + reshape1_input, name_suffix="_4D_" + ) + reshape1_output.shape = tflite_model.Shape(conv_2d_input_shape) + + reshape2_input = self.builder.duplicate_tensor( + t_op.tmp_outputs[0], name_suffix="_4D_" + ) + reshape2_input.shape = tflite_model.Shape(conv_2d_output_shape) + + reshape2_output = t_op.tmp_outputs[0] + + pre_reshapes = [] + + # Extend the weights tensor to 4D + weights_tensor = t_op.tmp_inputs[1] + if tensor_has_data(weights_tensor): + # Do it statically + weights_tensor.shape = tflite_model.Shape(conv_2d_weight_shape) + weights_tensor.tmp_buffer.data = weights_tensor.tmp_buffer.data.reshape( + conv_2d_weight_shape + ) + + else: + # Add a Reshape before the weights tensor + new_weights_tensor = self.builder.duplicate_tensor( + weights_tensor, name_suffix="_4D_" + ) + new_weights_tensor.shape = tflite_model.Shape(conv_2d_weight_shape) + + weight_reshape = tflite_model.Operator( + builtin_options=reshape_options.Reshape(conv_2d_weight_shape) + ) + weight_reshape.tmp_inputs = [weights_tensor] + weight_reshape.tmp_outputs = [new_weights_tensor] + + pre_reshapes.append(weight_reshape) + + # Save the new weights tensor, to assign it later. + weights_tensor = new_weights_tensor + + # -- Create the new operators -- + reshape1 = tflite_model.Operator( + builtin_options=reshape_options.Reshape(conv_2d_input_shape) + ) + reshape1.tmp_inputs = [reshape1_input] + reshape1.tmp_outputs = [reshape1_output] + pre_reshapes.append(reshape1) + + reshape2 = tflite_model.Operator( + builtin_options=reshape_options.Reshape(reshape2_output.shape.vector) + ) + reshape2.tmp_inputs = [reshape2_input] + reshape2.tmp_outputs = [reshape2_output] + + # Assign the new input and output of the Conv2D + t_op.tmp_inputs = [reshape1_output, weights_tensor] + t_op.tmp_inputs[ + 2: + ] # Add bias as well, if present + t_op.tmp_outputs = [reshape2_input] + + # Extend all Conv attributes to 2D + common.extend_1d_stride_to_2d(conv_params.stride) + common.extend_1d_dilation_to_2d(conv_params.dilation) + common.extend_1d_padding_to_2d(conv_params.padding) + + # Convert the now 2D Conv + converted_conv_ops = self._convert_2d_conv(t_op, conv_params) + + return pre_reshapes + converted_conv_ops + [reshape2] # noinspection PyPep8Naming def _convert_unpadded_2D( @@ -182,9 +289,19 @@ def _convert_2d_conv( aten_translator.convert_padding(conv_params.padding) ) if explicit_padding is not None: - # Need to prepend a 'Pad' operator, which adds 0s. + # Need to prepend a 'Pad' operator, which adds 0s (or `zero_point` for the quantized case). + input_quantization = t_op.tmp_inputs[0].quantization + pad_value = ( + None + if input_quantization is None + else np.array(input_quantization.zero_point[0]).astype( + tf_lite_type_to_numpy(t_op.tmp_inputs[0].type) + ) + ) conversion_result.ops_list.add_pre( - self.builder.create_pad_operator_before(t_op, 0, explicit_padding) + self.builder.create_pad_operator_before( + t_op, 0, explicit_padding, constant_value=pad_value + ) ) # DepthwiseConv2D expects weights in format [kernel_channels, kernel_height, kernel_width, output_channels] @@ -221,9 +338,19 @@ def _convert_2d_conv( aten_translator.convert_padding(conv_params.padding) ) if explicit_padding is not None: - # Need to prepend a 'Pad' operator, which adds 0s. + # Need to prepend a 'Pad' operator, which adds 0s (or `zero_point` for the quantized case). + input_quantization = t_op.tmp_inputs[0].quantization + pad_value = ( + None + if input_quantization is None + else np.array(input_quantization.zero_point[0]).astype( + tf_lite_type_to_numpy(t_op.tmp_inputs[0].type) + ) + ) conversion_result.ops_list.add_pre( - self.builder.create_pad_operator_before(t_op, 0, explicit_padding) + self.builder.create_pad_operator_before( + t_op, 0, explicit_padding, constant_value=pad_value + ) ) return conversion_result.ops_list.flatten() @@ -237,7 +364,9 @@ def convert(self, node: Node): conv_params = ConvParameters(stride, padding, dilation, groups) rank = t_op.tmp_inputs[1].shape.len() - if rank == 4: # Conv2D + if rank == 3: # Conv1D + ops_to_add = self._convert_1d_conv(t_op, conv_params) + elif rank == 4: # Conv2D ops_to_add = self._convert_2d_conv(t_op, conv_params) else: raise NotImplementedError( diff --git a/backends/nxp/backend/ir/converter/node_converters/shared/conv_utils.py b/backends/nxp/backend/ir/converter/node_converters/shared/conv_utils.py index ce03d4f6f15..3422e214982 100755 --- a/backends/nxp/backend/ir/converter/node_converters/shared/conv_utils.py +++ b/backends/nxp/backend/ir/converter/node_converters/shared/conv_utils.py @@ -14,6 +14,9 @@ ) from executorch.backends.nxp.backend.ir.converter.conversion import aten_translator from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList +from executorch.backends.nxp.backend.ir.converter.conversion.translator import ( + tf_lite_type_to_numpy, +) from executorch.backends.nxp.backend.ir.converter.tensor_utils import tensor_has_data from executorch.backends.nxp.backend.ir.lib.tflite.Padding import Padding from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model @@ -289,9 +292,17 @@ def build_input_tensor_padding( tfl_padding, explicit_padding = aten_translator.convert_padding(conv_params.padding) if explicit_padding is not None: - # Must add extra 'Pad' operator + # Must add extra 'Pad' operator, which adds 0s (or `zero_point` for the quantized case). + input_quantization = t_op.tmp_inputs[0].quantization + pad_value = ( + None + if input_quantization is None + else np.array(input_quantization.zero_point[0]).astype( + tf_lite_type_to_numpy(t_op.tmp_inputs[0].type) + ) + ) return tfl_padding, builder.create_pad_operator_before( - t_op, input_idx, explicit_padding + t_op, input_idx, explicit_padding, pad_value ) return tfl_padding, None diff --git a/backends/nxp/tests/executorch_pipeline.py b/backends/nxp/tests/executorch_pipeline.py index a426702cbba..96781ab6b10 100644 --- a/backends/nxp/tests/executorch_pipeline.py +++ b/backends/nxp/tests/executorch_pipeline.py @@ -48,7 +48,7 @@ def get_random_float_data(input_shapes: tuple[int] | list[tuple[int]]): def to_quantized_edge_program( model: torch.nn.Module, - input_shapes: tuple[int] | list[tuple[int]], + input_shapes: tuple[int, ...] | list[tuple[int, ...]], operators_not_to_delegate: list[str] = None, target="imxrt700", neutron_converter_flavor="SDK_25_03", @@ -100,7 +100,7 @@ def to_quantized_edge_program( def to_quantized_executorch_program( - model: torch.nn.Module, input_shapes: tuple[int] | list[tuple[int]] + model: torch.nn.Module, input_shapes: tuple[int, ...] | list[tuple[int, ...]] ) -> ExecutorchProgramManager: edge_program_manager = to_quantized_edge_program(model, input_shapes) @@ -110,7 +110,7 @@ def to_quantized_executorch_program( def to_edge_program( - model: nn.Module, input_shapes: tuple[int] | list[tuple[int]] + model: nn.Module, input_shapes: tuple[int, ...] | list[tuple[int, ...]] ) -> EdgeProgramManager: if isinstance(input_shapes, list): assert all(isinstance(input_shape, tuple) for input_shape in input_shapes), ( diff --git a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py index eb2818570f1..68550692049 100644 --- a/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py +++ b/backends/nxp/tests/ir/converter/node_converter/test_conv_converter.py @@ -1,4 +1,4 @@ -# Copyright 2024 NXP +# Copyright 2024-2025 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -22,10 +22,10 @@ ) from executorch.backends.nxp.tests.executors import ( convert_run_compare, - ToNCHWPreprocess, - ToNHWCPreprocess, + ToChannelFirstPreprocess, + ToChannelLastPreprocess, ) -from executorch.backends.nxp.tests.models import Conv2dModule +from executorch.backends.nxp.tests.models import Conv1dModule, Conv2dModule from torch.export import ExportedProgram @@ -35,49 +35,364 @@ def reseed_model_per_test_run(): np.random.seed(23) +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +def test_conv1d_quant_conversion(stride, dilation, kernel_size, mocker): + input_shape = (1, 4, 16) + model = Conv1dModule(stride=stride, dilation=dilation, kernel_size=kernel_size) + converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + _ = to_quantized_edge_program(model, input_shape) + + # Capture generated model + tflite_flatbuffers_model, io_formats = converter_spy.spy_return + + # Capture converted program + exported_program: ExportedProgram = converter_spy.call_args.args[1] + + input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8) + + convert_run_compare( + exported_program, + tflite_input_preprocess=ToChannelLastPreprocess(), + tfl_model=tflite_flatbuffers_model, + tflite_output_preprocess=ToChannelFirstPreprocess(), + input_data=input_data, + atol=1.0, + ) + + # Capture IR model ops + conversion_result = ops_spy.spy_return + ops = conversion_result.sub_graphs[0].operators.vector + + assert len(ops) == 3 + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.CONV_2D + assert ops[2].builtin_options.operator_type == BuiltinOperator.RESHAPE + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +@pytest.mark.parametrize("padding", [(1,), 2]) +def test_conv1d_quant_conversion__padded( + stride, dilation, kernel_size, padding, mocker +): + input_shape = (1, 4, 16) + model = Conv1dModule( + stride=stride, dilation=dilation, kernel_size=kernel_size, padding=padding + ) + converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + _ = to_quantized_edge_program(model, input_shape) + + # Capture generated model + tflite_flatbuffers_model, io_formats = converter_spy.spy_return + + # Capture converted program + exported_program: ExportedProgram = converter_spy.call_args.args[1] + + input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8) + + convert_run_compare( + exported_program, + tflite_input_preprocess=ToChannelLastPreprocess(), + tfl_model=tflite_flatbuffers_model, + tflite_output_preprocess=ToChannelFirstPreprocess(), + input_data=input_data, + atol=1.0, + ) + + # Capture IR model ops + conversion_result = ops_spy.spy_return + ops = conversion_result.sub_graphs[0].operators.vector + + assert len(ops) == 4 + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.PADV2 + assert ops[2].builtin_options.operator_type == BuiltinOperator.CONV_2D + assert ops[3].builtin_options.operator_type == BuiltinOperator.RESHAPE + + # Make sure the padding used the `zero-point`. + pad_value = ops[1].tmp_inputs[2].tmp_buffer.data.item() + assert ( + pad_value == ops[1].tmp_inputs[0].quantization.zero_point[0] + ) # `Pad` input zp. + assert ( + pad_value == ops[1].tmp_outputs[0].quantization.zero_point[0] + ) # `Pad` output zp. + assert ( + pad_value == ops[2].tmp_inputs[0].quantization.zero_point[0] + ) # `Conv` input zp. + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +def test_conv1d_quant_conversion__depthwise(stride, dilation, kernel_size, mocker): + input_shape = (1, 4, 16) + group = input_shape[1] + model = Conv1dModule( + group=group, + in_channels=group, + out_channels=group, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + ) + converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + _ = to_quantized_edge_program(model, input_shape) + + # Capture generated model + tflite_flatbuffers_model, io_formats = converter_spy.spy_return + + # Capture converted program + exported_program: ExportedProgram = converter_spy.call_args.args[1] + + input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8) + + convert_run_compare( + exported_program, + tflite_input_preprocess=ToChannelLastPreprocess(), + tfl_model=tflite_flatbuffers_model, + tflite_output_preprocess=ToChannelFirstPreprocess(), + input_data=input_data, + atol=1.0, + ) + + # Capture IR model ops + ops = ops_spy.spy_return.sub_graphs[0].operators.vector + + assert len(ops) == 3 + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.DEPTHWISE_CONV_2D + assert ops[2].builtin_options.operator_type == BuiltinOperator.RESHAPE + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +@pytest.mark.parametrize("padding", [(1,), 2]) +def test_conv1d_quant_conversion__depthwise__padded( + stride, dilation, kernel_size, padding, mocker +): + input_shape = (1, 4, 16) + group = input_shape[1] + model = Conv1dModule( + group=group, + in_channels=group, + out_channels=group, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + padding=padding, + ) + converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program") + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + _ = to_quantized_edge_program(model, input_shape) + + # Capture generated model + tflite_flatbuffers_model, io_formats = converter_spy.spy_return + + # Capture converted program + exported_program: ExportedProgram = converter_spy.call_args.args[1] + + input_data = (np.random.random(input_shape).astype(np.float32) * 50).astype(np.int8) + + convert_run_compare( + exported_program, + tflite_input_preprocess=ToChannelLastPreprocess(), + tfl_model=tflite_flatbuffers_model, + tflite_output_preprocess=ToChannelFirstPreprocess(), + input_data=input_data, + atol=1.0, + ) + + # Capture IR model ops + ops = ops_spy.spy_return.sub_graphs[0].operators.vector + + assert len(ops) == 4 + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.PADV2 + assert ops[2].builtin_options.operator_type == BuiltinOperator.DEPTHWISE_CONV_2D + assert ops[3].builtin_options.operator_type == BuiltinOperator.RESHAPE + + # Make sure the padding used the `zero-point`. + pad_value = ops[1].tmp_inputs[2].tmp_buffer.data.item() + assert ( + pad_value == ops[1].tmp_inputs[0].quantization.zero_point[0] + ) # `Pad` input zp. + assert ( + pad_value == ops[1].tmp_outputs[0].quantization.zero_point[0] + ) # `Pad` output zp. + assert ( + pad_value == ops[2].tmp_inputs[0].quantization.zero_point[0] + ) # `Conv` input zp. + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(3,), 4]) @pytest.mark.parametrize( - "input_shape, padding", - [ - pytest.param((1, 4, 32, 32), (0, 0), id="No padding."), - pytest.param( - (1, 4, 32, 32), - (1, 1), - id="Padding, keep the same output tensor size as input.", - ), - pytest.param( - (1, 4, 32, 32), (1, 0), id="Padding, change the output tensor size." - ), - pytest.param( - (1, 4, 31, 31), (1, 0), id="Padding, change the output tensor size." - ), - pytest.param( - (1, 4, 31, 31), (0, 1), id="Padding, change the output tensor size." - ), - ], + "input_shape, group, out_channels", [((1, 4, 12), 2, 2), ((1, 16, 9), 4, 16)] ) +def test_conv1d_conversion__separated( + input_shape, group, out_channels, stride, dilation, kernel_size, mocker +): + model = Conv1dModule( + group=group, + in_channels=input_shape[1], + out_channels=out_channels, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + ) + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + edge_program = to_edge_program(model, input_shape).exported_program() + + input_data = np.random.random(input_shape).astype(np.float32) + + convert_run_compare( + edge_program, + input_data, + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), + atol=3.0e-7, + ) + + # Capture IR model ops + ops = ops_spy.spy_return.sub_graphs[0].operators.vector + + assert ( + len(ops) == 1 + 1 + group + 1 + 1 + ) # Reshape + Split -> Conv (group times) -> Concat + Reshape + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.SPLIT + for op in ops[3:-2]: + assert op.builtin_options.operator_type == BuiltinOperator.CONV_2D + assert ops[-2].builtin_options.operator_type == BuiltinOperator.CONCATENATION + assert ops[-1].builtin_options.operator_type == BuiltinOperator.RESHAPE + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(3,), 4]) +@pytest.mark.parametrize("padding", [2, (1,)]) @pytest.mark.parametrize( - "dilation", - [ - pytest.param(1, id="No dilation."), - pytest.param(2, id="2 dilation."), - pytest.param((1, 3), id="Side-different dilation."), - ], + "input_shape, group, out_channels", [((1, 4, 12), 2, 2), ((1, 16, 9), 4, 16)] ) -def test_conv2d_conversion(input_shape, padding, dilation: int): - edge_program = to_edge_program( - Conv2dModule(padding=padding, dilation=dilation), input_shape - ).exported_program() +def test_conv1d_conversion__separated__padded( + input_shape, group, out_channels, stride, dilation, kernel_size, padding, mocker +): + model = Conv1dModule( + group=group, + in_channels=input_shape[1], + out_channels=out_channels, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + padding=padding, + ) + ops_spy = mocker.spy(ModelBuilder, "finish") + + # Run conversion + edge_program = to_edge_program(model, input_shape).exported_program() input_data = np.random.random(input_shape).astype(np.float32) convert_run_compare( edge_program, input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), - atol=4e-7, + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), + atol=3.0e-7, + ) + + # Capture IR model ops + ops = ops_spy.spy_return.sub_graphs[0].operators.vector + + assert ( + len(ops) == 1 + 1 + 2 * group + 1 + 1 + ) # Reshape + Split -> Pad + Conv (group times) -> Concat + Reshape + assert ops[0].builtin_options.operator_type == BuiltinOperator.RESHAPE + assert ops[1].builtin_options.operator_type == BuiltinOperator.SPLIT + for op in ops[2:-3:2]: + assert op.builtin_options.operator_type == BuiltinOperator.PAD + for op in ops[3:-2:2]: + assert op.builtin_options.operator_type == BuiltinOperator.CONV_2D + assert ops[-2].builtin_options.operator_type == BuiltinOperator.CONCATENATION + assert ops[-1].builtin_options.operator_type == BuiltinOperator.RESHAPE + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +@pytest.mark.parametrize( + "input_shape, group, out_channels", [((1, 4, 12), 2, 2), ((1, 16, 9), 4, 16)] +) +def test_conv1d_quant_conversion__separated( + input_shape, group, out_channels, stride, dilation, kernel_size +): + model = Conv1dModule( + group=group, + in_channels=input_shape[1], + out_channels=out_channels, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + ) + + # Run conversion + edge_program = to_quantized_edge_program(model, input_shape).exported_program() + + nodes = list(edge_program.graph.nodes) + assert len(nodes) == 11 + assert ( + nodes[7].target.__name__ == "aten.convolution.default" + ) # Convolution not delegated. + + +@pytest.mark.parametrize("stride", [1, 2]) +@pytest.mark.parametrize("dilation", [2, 1]) +@pytest.mark.parametrize("kernel_size", [(1,), (3,)]) +@pytest.mark.parametrize("padding", [(1,), 2]) +@pytest.mark.parametrize( + "input_shape, group, out_channels", [((1, 4, 12), 2, 2), ((1, 16, 9), 4, 16)] +) +def test_conv1d_quant_conversion__separated__padded( + input_shape, group, out_channels, stride, dilation, kernel_size, padding +): + model = Conv1dModule( + group=group, + in_channels=input_shape[1], + out_channels=out_channels, + stride=stride, + dilation=dilation, + kernel_size=kernel_size, + padding=padding, ) + # Run conversion + edge_program = to_quantized_edge_program(model, input_shape).exported_program() + + nodes = list(edge_program.graph.nodes) + assert len(nodes) == 11 + assert ( + nodes[7].target.__name__ == "aten.convolution.default" + ) # Convolution not delegated. + @pytest.mark.parametrize( "model, input_shape", @@ -204,9 +519,9 @@ def test_conv2d_quant_conversion(mocker, model: torch.nn.Module, input_shape): convert_run_compare( exported_program, - tflite_input_preprocess=ToNHWCPreprocess(), + tflite_input_preprocess=ToChannelLastPreprocess(), tfl_model=tflite_flatbuffers_model, - tflite_output_preprocess=ToNCHWPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), input_data=input_data, atol=1.0, ) @@ -237,8 +552,8 @@ def test_conv2d_conversion__depthwise(stride, dilation, kernel_shape, mocker): convert_run_compare( edge_program, input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), atol=4e-7, ) conversion_result = spy.spy_return @@ -299,8 +614,8 @@ def test_conv2d_conversion__depthwise__padded(padding, mocker): convert_run_compare( edge_program, input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), atol=4e-7, ) conversion_result = spy.spy_return @@ -326,7 +641,7 @@ def test_conv2d_conversion__depthwise__padded__quantized(padding, mocker): ops = spy.spy_return.sub_graphs[0].operators.vector assert len(ops) == 2 - assert ops[0].builtin_options.operator_type == BuiltinOperator.PAD + assert ops[0].builtin_options.operator_type == BuiltinOperator.PADV2 assert ops[1].builtin_options.operator_type == BuiltinOperator.DEPTHWISE_CONV_2D nodes = list(edge_program.graph.nodes) @@ -335,6 +650,12 @@ def test_conv2d_conversion__depthwise__padded__quantized(padding, mocker): ) # input, Quant, lowered_module, delegate_call, getitem, Deq, output assert nodes[2].target == "lowered_module_0" + # Make sure the padding used the `zero-point`. + assert ( + ops[0].tmp_inputs[2].tmp_buffer.data.item() + == ops[0].tmp_outputs[0].quantization.zero_point[0] + ) + @pytest.mark.parametrize("stride", [1, 2]) @pytest.mark.parametrize("dilation", [1, 2]) @@ -362,8 +683,8 @@ def test_conv2d_conversion__separated( convert_run_compare( edge_program, input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), atol=3.0e-7, ) @@ -442,8 +763,8 @@ def test_conv2d_conversion__separated__padded( convert_run_compare( edge_program, input_data, - tflite_input_preprocess=ToNHWCPreprocess(), - tflite_output_preprocess=ToNCHWPreprocess(), + tflite_input_preprocess=ToChannelLastPreprocess(), + tflite_output_preprocess=ToChannelFirstPreprocess(), atol=3.0e-7, ) diff --git a/backends/nxp/tests/models.py b/backends/nxp/tests/models.py index 19a253dccc8..80bba6a34df 100644 --- a/backends/nxp/tests/models.py +++ b/backends/nxp/tests/models.py @@ -9,6 +9,35 @@ import torch +class Conv1dModule(torch.nn.Module): + def __init__( + self, + bias: bool = True, + dilation: Union[int, tuple[int, int]] = 1, + in_channels: int = 4, + kernel_size: Union[int, tuple[int, int]] = 3, + out_channels: int = 8, + padding: Union[str, int, Collection[int]] = 0, + stride: Union[int, tuple[int, int]] = 2, + group: int = 1, + ): + super().__init__() + + self.conv = torch.nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + groups=group, + ) + + def forward(self, x): + return self.conv(x) + + class Conv2dModule(torch.nn.Module): def __init__( self,