nvidia-modeset: fix UB in DP DSC flatnessDetThresh calculation

src/nvidia-modeset/include/nvkms-utils.h

@@ -167,6 +167,9 @@ static inline NvU64 nvCtxDmaOffsetFromBytes(NvU64 ctxDmaOffset)
 
 NvU8 nvPixelDepthToBitsPerComponent(enum nvKmsPixelDepth pixelDepth);
 
+NvU32 nvDscPpsGetBitsPerComponent(const NvU32 *pps);
+NvU32 nvDscComputeFlatnessDetThresh(NvU32 bpc);
+
 typedef enum {
     EVO_LOG_WARN,
     EVO_LOG_ERROR,

src/nvidia-modeset/src/nvkms-evo3.c

@@ -7153,11 +7153,7 @@ static void EvoSetHdmiDscParams(const NVDispEvoRec *pDispEvo,
              pDscInfo->type == NV_DSC_INFO_EVO_TYPE_HDMI);
 
     bpc = nvPixelDepthToBitsPerComponent(pixelDepth);
-    if (bpc < 8) {
-        nvAssert(bpc >= 8);
-        bpc = 8;
-    }
-    flatnessDetThresh = (2 << (bpc - 8));
+    flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc);
 
     nvDmaSetStartEvoMethod(pChannel, NVC67D_HEAD_SET_DSC_CONTROL(head), 1);
     nvDmaSetEvoMethodData(pChannel,
@@ -7206,15 +7202,20 @@ static void EvoSetDpDscParams(const NVDispEvoRec *pDispEvo,
                               const NVDscInfoEvoRec *pDscInfo)
 {
     NVEvoChannelPtr pChannel = pDispEvo->pDevEvo->core;
-    NvU32 flatnessDetThresh;
+    NvU32 bpc, flatnessDetThresh;
     NvU32 i;
 
     nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP);
 
-    // XXX: I'm pretty sure that this is wrong.
-    // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is
-    // an insanely large number.
-    flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */
+    /*
+     * Extract bits_per_component from the PPS that was already computed
+     * by the DP library.  The DP path does not receive a pixelDepth
+     * parameter (unlike HDMI), so we read bpc from PPS byte 3 bits [7:4].
+     *
+     * Per VESA DSC 1.2a: flatness_det_thresh = 2 << (bpc - 8).
+     */
+    bpc = nvDscPpsGetBitsPerComponent(pDscInfo->dp.pps);
+    flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc);
 
     nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) ||
                 (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE));

src/nvidia-modeset/src/nvkms-evo4.c

@@ -1358,11 +1358,7 @@ static void EvoSetHdmiDscParamsC9(const NVDispEvoRec *pDispEvo,
              pDscInfo->type == NV_DSC_INFO_EVO_TYPE_HDMI);
 
     bpc = nvPixelDepthToBitsPerComponent(pixelDepth);
-    if (bpc < 8) {
-        nvAssert(bpc >= 8);
-        bpc = 8;
-    }
-    flatnessDetThresh = (2 << (bpc - 8));
+    flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc);
 
     nvDmaSetStartEvoMethod(pChannel, NVC97D_HEAD_SET_DSC_CONTROL(head), 1);
     nvDmaSetEvoMethodData(pChannel,
@@ -1408,15 +1404,20 @@ static void EvoSetDpDscParamsC9(const NVDispEvoRec *pDispEvo,
                               const NVDscInfoEvoRec *pDscInfo)
 {
     NVEvoChannelPtr pChannel = pDispEvo->pDevEvo->core;
-    NvU32 flatnessDetThresh;
+    NvU32 bpc, flatnessDetThresh;
     NvU32 i;
 
     nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP);
 
-    // XXX: I'm pretty sure that this is wrong.
-    // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is
-    // an insanely large number.
-    flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */
+    /*
+     * Extract bits_per_component from the PPS that was already computed
+     * by the DP library.  The DP path does not receive a pixelDepth
+     * parameter (unlike HDMI), so we read bpc from PPS byte 3 bits [7:4].
+     *
+     * Per VESA DSC 1.2a: flatness_det_thresh = 2 << (bpc - 8).
+     */
+    bpc = nvDscPpsGetBitsPerComponent(pDscInfo->dp.pps);
+    flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc);
 
     nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) ||
                 (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE));

src/nvidia-modeset/src/nvkms-utils.c

@@ -508,6 +508,52 @@ NvU8 nvPixelDepthToBitsPerComponent(enum nvKmsPixelDepth pixelDepth)
     return 0;
 }
 
+/*
+ * Extract bits_per_component from a DSC Picture Parameter Set (PPS).
+ *
+ * The PPS is an array of NvU32 dwords stored in native byte order.
+ * Per VESA DSC 1.2a Table 4-1, byte 3 bits [7:4] hold the
+ * bits_per_component value (8, 10, or 12 for typical displays).
+ * On little-endian machines byte 3 maps to pps[0] bits [31:28].
+ *
+ * Returns the raw bpc value from the PPS, or 0 if the field is
+ * reserved/zero.
+ */
+NvU32 nvDscPpsGetBitsPerComponent(const NvU32 *pps)
+{
+    /*
+     * PPS DWord 0 layout (bytes stored in native byte order):
+     *
+     *   bits [31:28] : bits_per_component   (PPS byte 3, bits [7:4])
+     *   bits [27:24] : linebuf_depth        (PPS byte 3, bits [3:0])
+     *   bits [23:16] : reserved             (PPS byte 2)
+     *   bits [15:8]  : pps_identifier       (PPS byte 1)
+     *   bits [7:4]   : dsc_version_major    (PPS byte 0, bits [7:4])
+     *   bits [3:0]   : dsc_version_minor    (PPS byte 0, bits [3:0])
+     */
+    return (pps[0] >> 28) & 0xF;
+}
+
+/*
+ * Compute the DSC flatness detection threshold from bits_per_component.
+ *
+ * Per VESA DSC 1.2a section 4.1:
+ *   flatness_det_thresh = 2 << (bpc - 8)
+ *
+ * bpc is expected to be >= 8 (the minimum the DSC standard supports).
+ * If bpc is unexpectedly below 8, the function clamps to the minimum
+ * valid threshold of 2 and fires an assertion.
+ */
+NvU32 nvDscComputeFlatnessDetThresh(NvU32 bpc)
+{
+    if (bpc < 8) {
+        nvAssert(bpc >= 8);
+        bpc = 8;
+    }
+
+    return (2 << (bpc - 8));
+}
+
 /* Import function required by nvBuildModeName() */
 
 int nvBuildModeNameSnprintf(char *str, size_t size, const char *format, ...)