Adding Test To Ensure All Future Quantizers Are Tested (#16099)

Summary:

We first create a list of quantizers that are currently not tested(we'll slowly reduce this to 0), and then we create a test to ensure that all future quantizers get tested using this framework. 

In order to do this, we needed to refactor how the current test is setup, specifically the parameterization.

Reviewed By: mcremon-meta, zonglinpeng, hsharma35

Differential Revision: D88055443

Author: RahulC7

backends/cadence/aot/tests/test_quantizer_ops.py

@@ -6,24 +6,32 @@
 
 # pyre-strict
 
+import inspect
 import unittest
 from typing import Callable
 
 import torch
 from executorch.backends.cadence.aot.graph_builder import GraphBuilder
+from executorch.backends.cadence.aot.quantizer import quantizer as quantizer_module
 from executorch.backends.cadence.aot.quantizer.patterns import AddmmPattern
+from executorch.exir.pass_base import NodeMetadata
 
 from executorch.backends.cadence.aot.quantizer.quantizer import (
     CadenceAtenQuantizer,
     CadenceDefaultQuantizer,
+    CadenceFusedConvReluQuantizer,
+    CadenceNopQuantizer,
     CadenceQuantizer,
     CadenceW8A32MixedQuantizer,
+    CadenceWakeWordQuantizer,
+    CadenceWith16BitConvActivationsQuantizer,
     CadenceWith16BitLinearActivationsQuantizer,
     CadenceWith16BitMatmulActivationsQuantizer,
+    CadenceWithLayerNormQuantizer,
+    CadenceWithSoftmaxQuantizer,
     qconfig_A16,
     qconfig_A8W8,
 )
-from executorch.exir.pass_base import NodeMetadata
 from parameterized import parameterized
 from torch._ops import OpOverload
 from torchao.quantization.pt2e.quantizer.quantizer import (
@@ -32,12 +40,67 @@
     QuantizationSpec,
 )
 
-# Type alias for graph builder functions
+# Type alias for graph builder functions.
+# These functions take a test instance and return a graph module and the target op node.
 GraphBuilderFn = Callable[
     ["QuantizerAnnotationTest"], tuple[torch.fx.GraphModule, torch.fx.Node]
 ]
 
 
+# Quantizers intentionally excluded from annotation testing.
+# These should be explicitly justified when added.
+EXCLUDED_FROM_ANNOTATION_TESTING: set[type[CadenceQuantizer]] = {
+    CadenceDefaultQuantizer,  # TODO: T247438143 Add test coverage
+    CadenceFusedConvReluQuantizer,  # TODO: T247438151 Add test coverage
+    CadenceNopQuantizer,  # No-op quantizer, doesn't annotate anything
+    CadenceW8A32MixedQuantizer,  # TODO: T247438158 Add test coverage
+    CadenceWakeWordQuantizer,  # TODO: T247438162 Add test coverage
+    CadenceWith16BitConvActivationsQuantizer,  # TODO: T247438221 Add test coverage
+    CadenceWithLayerNormQuantizer,  # TODO: T247438410 Add test coverage
+    CadenceWithSoftmaxQuantizer,  # TODO: T247438418 Add test coverage
+}
+
+
+# Test case definitions for quantizer annotation tests.
+# Format: (name, graph_builder_fn, quantizer_instance, target_op, expected_output_qspec, expected_input_qspecs)
+# Adding a new quantizer test only requires adding a tuple to this list.
+QUANTIZER_ANNOTATION_TEST_CASES: list[
+    tuple[
+        str,
+        GraphBuilderFn,
+        CadenceQuantizer,
+        OpOverload,
+        QuantizationSpec,
+        list[QuantizationSpec],
+    ]
+] = [
+    (
+        "matmul_A16",
+        lambda self: self._build_matmul_graph(),
+        CadenceWith16BitMatmulActivationsQuantizer(),
+        torch.ops.aten.matmul.default,
+        qconfig_A16.output_activation,
+        # For matmul, both inputs are activations
+        [qconfig_A16.input_activation, qconfig_A16.input_activation],
+    ),
+    (
+        "linear_A16",
+        lambda self: self._build_linear_graph(),
+        CadenceWith16BitLinearActivationsQuantizer(),
+        torch.ops.aten.linear.default,
+        qconfig_A16.output_activation,
+        # For linear: [input_activation, weight]
+        [qconfig_A16.input_activation, qconfig_A16.weight],
+    ),
+]
+
+# Derive the set of tested quantizer classes from the test cases.
+# This ensures TESTED_QUANTIZER_CLASSES stays in sync with actual tests.
+TESTED_QUANTIZER_CLASSES: set[type[CadenceQuantizer]] = {
+    type(case[2]) for case in QUANTIZER_ANNOTATION_TEST_CASES
+}
+
+
 class QuantizerAnnotationTest(unittest.TestCase):
     """Unit tests for verifying quantizer annotations are correctly applied."""
 
@@ -85,28 +148,7 @@ def _build_linear_graph(self) -> tuple[torch.fx.GraphModule, torch.fx.Node]:
         self.assertEqual(len(linear_nodes), 1, "Should find exactly one linear node")
         return gm, linear_nodes[0]
 
-    @parameterized.expand(
-        [
-            (
-                "matmul_A16",
-                lambda self: self._build_matmul_graph(),
-                CadenceWith16BitMatmulActivationsQuantizer(),
-                torch.ops.aten.matmul.default,
-                qconfig_A16.output_activation,
-                # For matmul, both inputs are activations
-                [qconfig_A16.input_activation, qconfig_A16.input_activation],
-            ),
-            (
-                "linear_A16",
-                lambda self: self._build_linear_graph(),
-                CadenceWith16BitLinearActivationsQuantizer(),
-                torch.ops.aten.linear.default,
-                qconfig_A16.output_activation,
-                # For linear: [input_activation, weight]
-                [qconfig_A16.input_activation, qconfig_A16.weight],
-            ),
-        ]
-    )
+    @parameterized.expand(QUANTIZER_ANNOTATION_TEST_CASES)
     def test_quantizer_annotation(
         self,
         name: str,
@@ -128,23 +170,45 @@ def test_quantizer_annotation(
         self.assertEqual(annotation.output_qspec, expected_output_qspec)
 
         # Verify input annotations
-        # Build actual_specs in the fixed order defined by op_node.args
         self.assertEqual(len(annotation.input_qspec_map), len(expected_input_qspecs))
-        actual_specs = [
-            annotation.input_qspec_map[op_node.args[i]]
-            for i in range(len(expected_input_qspecs))
-        ]
-
-        # Compare expected vs actual specs
-        for i, (expected, actual) in enumerate(
-            zip(expected_input_qspecs, actual_specs)
+        for i, (input_node, input_qspec) in enumerate(
+            annotation.input_qspec_map.items()
         ):
             self.assertEqual(
-                actual,
-                expected,
+                input_node,
+                op_node.args[i],
+                f"Input node mismatch at index {i}",
+            )
+            self.assertEqual(
+                input_qspec,
+                expected_input_qspecs[i],
                 f"Input qspec mismatch at index {i}",
             )
 
+    def test_all_quantizers_have_annotation_tests(self) -> None:
+        """Ensure every CadenceQuantizer subclass is either tested or explicitly excluded."""
+        # Get all CadenceQuantizer subclasses defined in the quantizer module
+        all_quantizers: set[type[CadenceQuantizer]] = set()
+        for _, obj in inspect.getmembers(quantizer_module, inspect.isclass):
+            if (
+                issubclass(obj, CadenceQuantizer)
+                and obj is not CadenceQuantizer
+                and obj.__module__ == quantizer_module.__name__
+            ):
+                all_quantizers.add(obj)
+
+        # Check for missing tests
+        untested = (
+            all_quantizers - TESTED_QUANTIZER_CLASSES - EXCLUDED_FROM_ANNOTATION_TESTING
+        )
+        if untested:
+            untested_names = sorted(cls.__name__ for cls in untested)
+            self.fail(
+                f"The following CadenceQuantizer subclasses are not tested in "
+                f"test_quantizer_annotation and not in EXCLUDED_FROM_ANNOTATION_TESTING: "
+                f"{untested_names}. Please add test cases or explicitly exclude them."
+            )
+
 
 class QuantizerOpsPreserveTest(unittest.TestCase):
     def test_mixed_w8a32_ops_to_preserve(self) -> None: