From 29a023739ebd3b5511bcff95665111b8cecdfd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A8=B1=E5=85=83=E8=B1=AA?= Date: Wed, 11 Mar 2026 10:45:15 +0800 Subject: [PATCH] nvidia-modeset: fix undefined behavior in DP DSC flatnessDetThresh calculation EvoSetDpDscParams() and EvoSetDpDscParamsC9() compute the DSC flatness detection threshold using bitsPerPixelX16 instead of bits_per_component. bitsPerPixelX16 is BPP * 16 (e.g. 128 for 8.0 bpp), so the expression 2 << (bitsPerPixelX16 - 8) produces a shift of 120+ bits, which is undefined behavior in C. The resulting garbage value is written to the FLATNESS_DET_THRESH hardware register field (10-bit). The HDMI DSC path already handles this correctly by deriving bpc from the pixelDepth parameter, but the DP path does not receive pixelDepth. Instead, extract bits_per_component from PPS byte 3 bits [7:4], which the DP library has already computed correctly. Introduce two shared helpers: - nvDscPpsGetBitsPerComponent(): extract bpc from PPS DWord 0 - nvDscComputeFlatnessDetThresh(): compute 2 << (bpc - 8) with validation and clamping for bpc < 8 Both DP and HDMI DSC paths now use nvDscComputeFlatnessDetThresh(), eliminating the duplicated inline validation in the HDMI path and ensuring consistent behavior across all display class versions. Fixes: VESA DSC 1.2a section 4.1 flatness_det_thresh formula Closes: #1029 --- src/nvidia-modeset/include/nvkms-utils.h | 3 ++ src/nvidia-modeset/src/nvkms-evo3.c | 21 +++++------ src/nvidia-modeset/src/nvkms-evo4.c | 21 +++++------ src/nvidia-modeset/src/nvkms-utils.c | 46 ++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/nvidia-modeset/include/nvkms-utils.h b/src/nvidia-modeset/include/nvkms-utils.h index 1d7db5fc8..3228913a1 100644 --- a/src/nvidia-modeset/include/nvkms-utils.h +++ b/src/nvidia-modeset/include/nvkms-utils.h @@ -167,6 +167,9 @@ static inline NvU64 nvCtxDmaOffsetFromBytes(NvU64 ctxDmaOffset) NvU8 nvPixelDepthToBitsPerComponent(enum nvKmsPixelDepth pixelDepth); +NvU32 nvDscPpsGetBitsPerComponent(const NvU32 *pps); +NvU32 nvDscComputeFlatnessDetThresh(NvU32 bpc); + typedef enum { EVO_LOG_WARN, EVO_LOG_ERROR, diff --git a/src/nvidia-modeset/src/nvkms-evo3.c b/src/nvidia-modeset/src/nvkms-evo3.c index 455836e80..84f4f0eef 100644 --- a/src/nvidia-modeset/src/nvkms-evo3.c +++ b/src/nvidia-modeset/src/nvkms-evo3.c @@ -7153,11 +7153,7 @@ static void EvoSetHdmiDscParams(const NVDispEvoRec *pDispEvo, pDscInfo->type == NV_DSC_INFO_EVO_TYPE_HDMI); bpc = nvPixelDepthToBitsPerComponent(pixelDepth); - if (bpc < 8) { - nvAssert(bpc >= 8); - bpc = 8; - } - flatnessDetThresh = (2 << (bpc - 8)); + flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc); nvDmaSetStartEvoMethod(pChannel, NVC67D_HEAD_SET_DSC_CONTROL(head), 1); nvDmaSetEvoMethodData(pChannel, @@ -7206,15 +7202,20 @@ static void EvoSetDpDscParams(const NVDispEvoRec *pDispEvo, const NVDscInfoEvoRec *pDscInfo) { NVEvoChannelPtr pChannel = pDispEvo->pDevEvo->core; - NvU32 flatnessDetThresh; + NvU32 bpc, flatnessDetThresh; NvU32 i; nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP); - // XXX: I'm pretty sure that this is wrong. - // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is - // an insanely large number. - flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */ + /* + * Extract bits_per_component from the PPS that was already computed + * by the DP library. The DP path does not receive a pixelDepth + * parameter (unlike HDMI), so we read bpc from PPS byte 3 bits [7:4]. + * + * Per VESA DSC 1.2a: flatness_det_thresh = 2 << (bpc - 8). + */ + bpc = nvDscPpsGetBitsPerComponent(pDscInfo->dp.pps); + flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc); nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) || (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE)); diff --git a/src/nvidia-modeset/src/nvkms-evo4.c b/src/nvidia-modeset/src/nvkms-evo4.c index a2b94b16e..890aeaf3b 100644 --- a/src/nvidia-modeset/src/nvkms-evo4.c +++ b/src/nvidia-modeset/src/nvkms-evo4.c @@ -1358,11 +1358,7 @@ static void EvoSetHdmiDscParamsC9(const NVDispEvoRec *pDispEvo, pDscInfo->type == NV_DSC_INFO_EVO_TYPE_HDMI); bpc = nvPixelDepthToBitsPerComponent(pixelDepth); - if (bpc < 8) { - nvAssert(bpc >= 8); - bpc = 8; - } - flatnessDetThresh = (2 << (bpc - 8)); + flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc); nvDmaSetStartEvoMethod(pChannel, NVC97D_HEAD_SET_DSC_CONTROL(head), 1); nvDmaSetEvoMethodData(pChannel, @@ -1408,15 +1404,20 @@ static void EvoSetDpDscParamsC9(const NVDispEvoRec *pDispEvo, const NVDscInfoEvoRec *pDscInfo) { NVEvoChannelPtr pChannel = pDispEvo->pDevEvo->core; - NvU32 flatnessDetThresh; + NvU32 bpc, flatnessDetThresh; NvU32 i; nvAssert(pDscInfo->type == NV_DSC_INFO_EVO_TYPE_DP); - // XXX: I'm pretty sure that this is wrong. - // BitsPerPixelx16 is something like (24 * 16) = 384, and 2 << (384 - 8) is - // an insanely large number. - flatnessDetThresh = (2 << (pDscInfo->dp.bitsPerPixelX16 - 8)); /* ??? */ + /* + * Extract bits_per_component from the PPS that was already computed + * by the DP library. The DP path does not receive a pixelDepth + * parameter (unlike HDMI), so we read bpc from PPS byte 3 bits [7:4]. + * + * Per VESA DSC 1.2a: flatness_det_thresh = 2 << (bpc - 8). + */ + bpc = nvDscPpsGetBitsPerComponent(pDscInfo->dp.pps); + flatnessDetThresh = nvDscComputeFlatnessDetThresh(bpc); nvAssert((pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_DUAL) || (pDscInfo->dp.dscMode == NV_DSC_EVO_MODE_SINGLE)); diff --git a/src/nvidia-modeset/src/nvkms-utils.c b/src/nvidia-modeset/src/nvkms-utils.c index 9ac6b800a..055e1757a 100644 --- a/src/nvidia-modeset/src/nvkms-utils.c +++ b/src/nvidia-modeset/src/nvkms-utils.c @@ -508,6 +508,52 @@ NvU8 nvPixelDepthToBitsPerComponent(enum nvKmsPixelDepth pixelDepth) return 0; } +/* + * Extract bits_per_component from a DSC Picture Parameter Set (PPS). + * + * The PPS is an array of NvU32 dwords stored in native byte order. + * Per VESA DSC 1.2a Table 4-1, byte 3 bits [7:4] hold the + * bits_per_component value (8, 10, or 12 for typical displays). + * On little-endian machines byte 3 maps to pps[0] bits [31:28]. + * + * Returns the raw bpc value from the PPS, or 0 if the field is + * reserved/zero. + */ +NvU32 nvDscPpsGetBitsPerComponent(const NvU32 *pps) +{ + /* + * PPS DWord 0 layout (bytes stored in native byte order): + * + * bits [31:28] : bits_per_component (PPS byte 3, bits [7:4]) + * bits [27:24] : linebuf_depth (PPS byte 3, bits [3:0]) + * bits [23:16] : reserved (PPS byte 2) + * bits [15:8] : pps_identifier (PPS byte 1) + * bits [7:4] : dsc_version_major (PPS byte 0, bits [7:4]) + * bits [3:0] : dsc_version_minor (PPS byte 0, bits [3:0]) + */ + return (pps[0] >> 28) & 0xF; +} + +/* + * Compute the DSC flatness detection threshold from bits_per_component. + * + * Per VESA DSC 1.2a section 4.1: + * flatness_det_thresh = 2 << (bpc - 8) + * + * bpc is expected to be >= 8 (the minimum the DSC standard supports). + * If bpc is unexpectedly below 8, the function clamps to the minimum + * valid threshold of 2 and fires an assertion. + */ +NvU32 nvDscComputeFlatnessDetThresh(NvU32 bpc) +{ + if (bpc < 8) { + nvAssert(bpc >= 8); + bpc = 8; + } + + return (2 << (bpc - 8)); +} + /* Import function required by nvBuildModeName() */ int nvBuildModeNameSnprintf(char *str, size_t size, const char *format, ...)