From 403e04e4798c430009215b1e5feaca1deaf87a7f Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Tue, 23 Dec 2025 20:36:22 +0200 Subject: [PATCH 1/2] Add platform-specific frame utilities and optimize VideoPlayerState frame processing --- .../linux/LinuxFrameUtils.kt | 90 ++++++++ .../linux/LinuxVideoPlayerState.kt | 101 ++++++--- .../composemediaplayer/mac/MacFrameUtils.kt | 66 ++++++ .../mac/MacVideoPlayerState.kt | 166 +++++++------- .../windows/WindowsFrameUtils.kt | 90 ++++++++ .../windows/WindowsVideoPlayerState.kt | 205 +++++++++++------- .../mac/MacFrameUtilsTest.kt | 113 ++++++++++ 7 files changed, 655 insertions(+), 176 deletions(-) create mode 100644 mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxFrameUtils.kt create mode 100644 mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtils.kt create mode 100644 mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsFrameUtils.kt create mode 100644 mediaplayer/src/jvmTest/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtilsTest.kt diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxFrameUtils.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxFrameUtils.kt new file mode 100644 index 00000000..b41c47a7 --- /dev/null +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxFrameUtils.kt @@ -0,0 +1,90 @@ +package io.github.kdroidfilter.composemediaplayer.linux + +import java.nio.ByteBuffer + +/** + * Calculates a fast hash of the frame buffer to detect frame changes. + * Samples approximately 200 pixels evenly distributed across the frame. + * + * @param buffer The source buffer containing RGBA pixel data + * @param pixelCount Total number of pixels in the frame + * @return A hash value representing the frame content + */ +internal fun calculateFrameHash(buffer: ByteBuffer, pixelCount: Int): Int { + if (pixelCount <= 0) return 0 + + var hash = 1 + val step = if (pixelCount <= 200) 1 else pixelCount / 200 + for (i in 0 until pixelCount step step) { + hash = 31 * hash + buffer.getInt(i * 4) + } + return hash +} + +/** + * Copies RGBA frame data from source to destination buffer with minimal overhead. + * Handles row padding when destination stride differs from source. + * + * This function performs a single memory copy operation when strides match, + * achieving zero-copy performance (beyond the necessary single copy from + * GStreamer buffer to Skia bitmap). + * + * @param src Source buffer containing RGBA pixel data from GStreamer + * @param dst Destination buffer (Skia bitmap pixels via peekPixels) + * @param width Frame width in pixels + * @param height Frame height in pixels + * @param dstRowBytes Destination row stride (may include padding) + */ +internal fun copyRgbaFrame( + src: ByteBuffer, + dst: ByteBuffer, + width: Int, + height: Int, + dstRowBytes: Int, +) { + require(width > 0) { "width must be > 0 (was $width)" } + require(height > 0) { "height must be > 0 (was $height)" } + val srcRowBytes = width * 4 + require(dstRowBytes >= srcRowBytes) { + "dstRowBytes ($dstRowBytes) must be >= srcRowBytes ($srcRowBytes)" + } + + val requiredSrcBytes = srcRowBytes.toLong() * height.toLong() + val requiredDstBytes = dstRowBytes.toLong() * height.toLong() + require(src.capacity().toLong() >= requiredSrcBytes) { + "src buffer too small: ${src.capacity()} < $requiredSrcBytes" + } + require(dst.capacity().toLong() >= requiredDstBytes) { + "dst buffer too small: ${dst.capacity()} < $requiredDstBytes" + } + + val srcBuf = src.duplicate() + val dstBuf = dst.duplicate() + srcBuf.rewind() + dstBuf.rewind() + + // Fast path: when strides match, do a single bulk copy + if (dstRowBytes == srcRowBytes) { + srcBuf.limit(requiredSrcBytes.toInt()) + dstBuf.limit(requiredSrcBytes.toInt()) + dstBuf.put(srcBuf) + return + } + + // Slow path: copy row by row when there's padding + val srcCapacity = srcBuf.capacity() + val dstCapacity = dstBuf.capacity() + for (row in 0 until height) { + val srcPos = row * srcRowBytes + srcBuf.limit(srcCapacity) + srcBuf.position(srcPos) + srcBuf.limit(srcPos + srcRowBytes) + + val dstPos = row * dstRowBytes + dstBuf.limit(dstCapacity) + dstBuf.position(dstPos) + dstBuf.limit(dstPos + srcRowBytes) + + dstBuf.put(srcBuf) + } +} diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxVideoPlayerState.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxVideoPlayerState.kt index 508da0c9..9b939c35 100644 --- a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxVideoPlayerState.kt +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/linux/LinuxVideoPlayerState.kt @@ -25,6 +25,7 @@ import org.freedesktop.gstreamer.Format import org.freedesktop.gstreamer.event.SeekFlags import org.freedesktop.gstreamer.event.SeekType import org.freedesktop.gstreamer.message.MessageType +import com.sun.jna.Pointer import org.jetbrains.skia.Bitmap import org.jetbrains.skia.ColorAlphaType import org.jetbrains.skia.ColorType @@ -32,6 +33,7 @@ import org.jetbrains.skia.ImageInfo import java.awt.EventQueue import java.io.File import java.net.URI +import java.nio.ByteBuffer import java.util.EnumSet import javax.swing.Timer import kotlin.math.abs @@ -71,6 +73,12 @@ class LinuxVideoPlayerState : PlatformVideoPlayerState { private var frameWidth = 0 private var frameHeight = 0 + // Double-buffering for zero-copy frame rendering + private var skiaBitmapA: Bitmap? = null + private var skiaBitmapB: Bitmap? = null + private var nextSkiaBitmapA: Boolean = true + private var lastFrameHash: Int = Int.MIN_VALUE + private var bufferingPercent by mutableStateOf(100) private var isUserPaused by mutableStateOf(false) private var hasReceivedFirstFrame by mutableStateOf(false) @@ -776,6 +784,7 @@ class LinuxVideoPlayerState : PlatformVideoPlayerState { _isSeeking = false hasReceivedFirstFrame = false _currentFrame = null + lastFrameHash = Int.MIN_VALUE } override fun seekTo(value: Float) { @@ -813,10 +822,16 @@ class LinuxVideoPlayerState : PlatformVideoPlayerState { // ---- Processing of a video sample ---- /** - * Directly reads in RGBA and copies to a Skia Bitmap in the RGBA_8888 format - * (non-premultiplied). This avoids redundant conversions to maintain accurate colors and performance. - * - * Optimized for better performance, especially in fullscreen mode. + * Zero-copy optimized frame processing using double-buffering and direct memory access. + * + * Optimizations applied: + * 1. Double-buffering: Reuses two Bitmap objects, alternating between them to avoid + * allocating new bitmaps every frame while the UI draws from the previous one. + * 2. Frame hashing: Skips processing if the frame content hasn't changed (identical frames). + * 3. peekPixels(): Direct access to Skia bitmap memory, avoiding intermediate ByteArray allocation. + * 4. Single memory copy: GStreamer buffer → Skia bitmap pixels (true zero-copy beyond this necessary transfer). + * + * Memory flow: GStreamer native buffer → Skia bitmap pixels (1 copy via bulk ByteBuffer.put) */ private fun processSample(sample: Sample) { try { @@ -826,40 +841,66 @@ class LinuxVideoPlayerState : PlatformVideoPlayerState { val width = structure.getInteger("width") val height = structure.getInteger("height") + if (width <= 0 || height <= 0) return + + // Handle dimension changes if (width != frameWidth || height != frameHeight) { frameWidth = width frameHeight = height + + // Reallocate bitmaps for new dimensions + skiaBitmapA?.close() + skiaBitmapB?.close() + + val imageInfo = ImageInfo(width, height, ColorType.RGBA_8888, ColorAlphaType.UNPREMUL) + skiaBitmapA = Bitmap().apply { allocPixels(imageInfo) } + skiaBitmapB = Bitmap().apply { allocPixels(imageInfo) } + nextSkiaBitmapA = true + lastFrameHash = Int.MIN_VALUE + updateAspectRatio() } val buffer = sample.buffer ?: return - val byteBuffer = buffer.map(false) ?: return - byteBuffer.rewind() - - // Prepare a Skia Bitmap - val imageInfo = ImageInfo( - width, - height, - ColorType.RGBA_8888, - ColorAlphaType.UNPREMUL - ) + val srcBuffer = buffer.map(false) ?: return + srcBuffer.rewind() + + val pixelCount = width * height - val bitmap = Bitmap() - bitmap.allocPixels(imageInfo) + // Calculate frame hash to detect identical frames + val newHash = calculateFrameHash(srcBuffer, pixelCount) + if (newHash == lastFrameHash) { + buffer.unmap() + return + } + lastFrameHash = newHash + + // Select the target bitmap (double-buffering) + val targetBitmap = if (nextSkiaBitmapA) skiaBitmapA!! else skiaBitmapB!! + nextSkiaBitmapA = !nextSkiaBitmapA - // Get the byte array from the buffer directly - val totalBytes = width * height * 4 - val byteArray = ByteArray(totalBytes) + // Get direct access to bitmap pixels via peekPixels (zero-copy access) + val pixmap = targetBitmap.peekPixels() ?: run { + buffer.unmap() + return + } - // Bulk copy the bytes from the buffer to the array - // This is much more efficient than copying pixel by pixel - byteBuffer.get(byteArray, 0, totalBytes) + val pixelsAddr = pixmap.addr + if (pixelsAddr == 0L) { + buffer.unmap() + return + } - // Install these pixels into the Bitmap - bitmap.installPixels(imageInfo, byteArray, width * 4) + // Single memory copy: GStreamer buffer → Skia bitmap + val dstRowBytes = pixmap.rowBytes.toInt() + val dstSizeBytes = dstRowBytes.toLong() * height.toLong() + val dstBuffer = Pointer(pixelsAddr).getByteBuffer(0, dstSizeBytes) - // Convert the Skia Bitmap into a Compose ImageBitmap - val imageBitmap = bitmap.asComposeImageBitmap() + srcBuffer.rewind() + copyRgbaFrame(srcBuffer, dstBuffer, width, height, dstRowBytes) + + // Convert to Compose ImageBitmap + val imageBitmap = targetBitmap.asComposeImageBitmap() // Update on the AWT thread EventQueue.invokeLater { @@ -884,6 +925,14 @@ class LinuxVideoPlayerState : PlatformVideoPlayerState { playbin.stop() playbin.dispose() videoSink.dispose() + + // Clean up double-buffering bitmaps + skiaBitmapA?.close() + skiaBitmapB?.close() + skiaBitmapA = null + skiaBitmapB = null + lastFrameHash = Int.MIN_VALUE + // Don't call Gst.deinit() here as it would affect all instances // Each instance should only clean up its own resources } diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtils.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtils.kt new file mode 100644 index 00000000..e6a9f6e3 --- /dev/null +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtils.kt @@ -0,0 +1,66 @@ +package io.github.kdroidfilter.composemediaplayer.mac + +import java.nio.ByteBuffer + +internal fun calculateFrameHash(buffer: ByteBuffer, pixelCount: Int): Int { + if (pixelCount <= 0) return 0 + + var hash = 1 + val step = if (pixelCount <= 200) 1 else pixelCount / 200 + for (i in 0 until pixelCount step step) { + hash = 31 * hash + buffer.getInt(i * 4) + } + return hash +} + +internal fun copyBgraFrame( + src: ByteBuffer, + dst: ByteBuffer, + width: Int, + height: Int, + dstRowBytes: Int, +) { + require(width > 0) { "width must be > 0 (was $width)" } + require(height > 0) { "height must be > 0 (was $height)" } + val srcRowBytes = width * 4 + require(dstRowBytes >= srcRowBytes) { + "dstRowBytes ($dstRowBytes) must be >= srcRowBytes ($srcRowBytes)" + } + + val requiredSrcBytes = srcRowBytes.toLong() * height.toLong() + val requiredDstBytes = dstRowBytes.toLong() * height.toLong() + require(src.capacity().toLong() >= requiredSrcBytes) { + "src buffer too small: ${src.capacity()} < $requiredSrcBytes" + } + require(dst.capacity().toLong() >= requiredDstBytes) { + "dst buffer too small: ${dst.capacity()} < $requiredDstBytes" + } + + val srcBuf = src.duplicate() + val dstBuf = dst.duplicate() + srcBuf.rewind() + dstBuf.rewind() + + if (dstRowBytes == srcRowBytes) { + srcBuf.limit(requiredSrcBytes.toInt()) + dstBuf.limit(requiredSrcBytes.toInt()) + dstBuf.put(srcBuf) + return + } + + val srcCapacity = srcBuf.capacity() + val dstCapacity = dstBuf.capacity() + for (row in 0 until height) { + val srcPos = row * srcRowBytes + srcBuf.limit(srcCapacity) + srcBuf.position(srcPos) + srcBuf.limit(srcPos + srcRowBytes) + + val dstPos = row * dstRowBytes + dstBuf.limit(dstCapacity) + dstBuf.position(dstPos) + dstBuf.limit(dstPos + srcRowBytes) + + dstBuf.put(srcBuf) + } +} diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacVideoPlayerState.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacVideoPlayerState.kt index 936fb4b3..99e31fce 100644 --- a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacVideoPlayerState.kt +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacVideoPlayerState.kt @@ -3,7 +3,7 @@ package io.github.kdroidfilter.composemediaplayer.mac import androidx.compose.runtime.* import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.ImageBitmap -import androidx.compose.ui.graphics.toComposeImageBitmap +import androidx.compose.ui.graphics.asComposeImageBitmap import androidx.compose.ui.text.TextStyle import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextAlign @@ -25,9 +25,11 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.debounce import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import java.awt.image.BufferedImage -import java.awt.image.DataBufferInt import java.net.URI +import org.jetbrains.skia.Bitmap +import org.jetbrains.skia.ColorAlphaType +import org.jetbrains.skia.ColorType +import org.jetbrains.skia.ImageInfo import kotlin.math.abs import kotlin.math.log10 @@ -45,10 +47,15 @@ class MacVideoPlayerState : PlatformVideoPlayerState { // Main state variables private val mainMutex = Mutex() + private val frameMutex = Mutex() private var playerPtr: Pointer? = null private val _currentFrameState = MutableStateFlow(null) internal val currentFrameState: State = mutableStateOf(null) - private var _bufferImage: BufferedImage? = null + private var skiaBitmapWidth: Int = 0 + private var skiaBitmapHeight: Int = 0 + private var skiaBitmapA: Bitmap? = null + private var skiaBitmapB: Bitmap? = null + private var nextSkiaBitmapA: Boolean = true // Audio level state variables (added for left and right levels) private val _leftLevel = mutableStateOf(0.0f) @@ -67,7 +74,7 @@ class MacVideoPlayerState : PlatformVideoPlayerState { private var lastFrameUpdateTime: Long = 0 private var seekInProgress = false private var targetSeekTime: Double? = null - private var lastFrameHash: Int = 0 + private var lastFrameHash: Int = Int.MIN_VALUE private var videoFrameRate: Float = 0.0f private var screenRefreshRate: Float = 0.0f private var captureFrameRate: Float = 0.0f @@ -322,10 +329,13 @@ class MacVideoPlayerState : PlatformVideoPlayerState { stopFrameUpdates() stopBufferingCheck() - val ptrToDispose = mainMutex.withLock { - val ptr = playerPtr - playerPtr = null - ptr + val ptrToDispose = frameMutex.withLock { + lastFrameHash = Int.MIN_VALUE + mainMutex.withLock { + val ptr = playerPtr + playerPtr = null + ptr + } } // Release resources outside of the mutex lock @@ -524,71 +534,70 @@ class MacVideoPlayerState : PlatformVideoPlayerState { bufferingCheckJob = null } - /** - * Calculates a simple hash of the image data to detect if the frame has - * changed. This runs on the compute dispatcher for CPU-intensive work and - * samples fewer pixels for better performance. - */ - private suspend fun calculateFrameHash(data: IntArray): Int = withContext(Dispatchers.Default) { - var hash = 0 - // Sample a smaller subset of pixels for performance - val step = data.size / 200 - if (step > 0) { - for (i in data.indices step step) { - hash = 31 * hash + data[i] - } - } - hash - } - /** Updates the current video frame on a background thread. */ private suspend fun updateFrameAsync() { - try { - // Safely get the player pointer - val ptr = mainMutex.withLock { playerPtr } ?: return + frameMutex.withLock { + try { + // Safely get the player pointer + val ptr = mainMutex.withLock { playerPtr } ?: return - // Get frame dimensions - val width = SharedVideoPlayer.getFrameWidth(ptr) - val height = SharedVideoPlayer.getFrameHeight(ptr) + // Get frame dimensions + val width = SharedVideoPlayer.getFrameWidth(ptr) + val height = SharedVideoPlayer.getFrameHeight(ptr) - if (width <= 0 || height <= 0) { - return - } + if (width <= 0 || height <= 0) { + return + } - // Get the latest frame to minimize mutex lock time - val framePtr = SharedVideoPlayer.getLatestFrame(ptr) ?: return + // Get the latest frame to minimize mutex lock time + val framePtr = SharedVideoPlayer.getLatestFrame(ptr) ?: return - // Create or reuse a buffered image on a compute thread - val bufferedImage = withContext(Dispatchers.Default) { - val existingBuffer = mainMutex.withLock { _bufferImage } - if (existingBuffer == null || existingBuffer.width != width || existingBuffer.height != height) { - val newBuffer = BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB) - mainMutex.withLock { _bufferImage = newBuffer } - newBuffer - } else { - existingBuffer - } - } + val pixelCount = width * height + val frameSizeBytes = pixelCount.toLong() * 4L + var framePublished = false - // Copy frame data on a compute thread for performance - withContext(Dispatchers.Default) { - val pixels = (bufferedImage.raster.dataBuffer as DataBufferInt).data - framePtr.getByteBuffer(0, (width * height * 4).toLong()).asIntBuffer().get(pixels) + withContext(Dispatchers.Default) { + val srcBuf = framePtr.getByteBuffer(0, frameSizeBytes) - // Calculate frame hash to detect changes - val newHash = calculateFrameHash(pixels) - val frameChanged = newHash != lastFrameHash - lastFrameHash = newHash + // Calculate a simple hash to avoid redundant copies/conversions. + val newHash = calculateFrameHash(srcBuf, pixelCount) + if (newHash == lastFrameHash) return@withContext + lastFrameHash = newHash - if (frameChanged) { - // Update timestamp - lastFrameUpdateTime = System.currentTimeMillis() + // Allocate/reuse two bitmaps (double-buffering) to avoid writing while the UI draws. + if (skiaBitmapA == null || skiaBitmapWidth != width || skiaBitmapHeight != height) { + skiaBitmapA?.close() + skiaBitmapB?.close() + + val imageInfo = ImageInfo(width, height, ColorType.BGRA_8888, ColorAlphaType.OPAQUE) + skiaBitmapA = Bitmap().apply { allocPixels(imageInfo) } + skiaBitmapB = Bitmap().apply { allocPixels(imageInfo) } + skiaBitmapWidth = width + skiaBitmapHeight = height + nextSkiaBitmapA = true + } + + val targetBitmap = if (nextSkiaBitmapA) skiaBitmapA!! else skiaBitmapB!! + nextSkiaBitmapA = !nextSkiaBitmapA + + val pixmap = targetBitmap.peekPixels() ?: return@withContext + val pixelsAddr = pixmap.addr + if (pixelsAddr == 0L) return@withContext - // Convert to ImageBitmap on a compute thread - val imageBitmap = bufferedImage.toComposeImageBitmap() + // Native-to-native copy: frame buffer (JNA) -> Skia bitmap pixels. + srcBuf.rewind() + val destRowBytes = pixmap.rowBytes.toInt() + val destSizeBytes = destRowBytes.toLong() * height.toLong() + val destBuf = Pointer(pixelsAddr).getByteBuffer(0, destSizeBytes) + copyBgraFrame(srcBuf, destBuf, width, height, destRowBytes) // Publish to flow - _currentFrameState.value = imageBitmap + _currentFrameState.value = targetBitmap.asComposeImageBitmap() + framePublished = true + } + + if (framePublished) { + lastFrameUpdateTime = System.currentTimeMillis() // Update loading state if needed on the main thread if (isLoading && !seekInProgress) { @@ -597,10 +606,10 @@ class MacVideoPlayerState : PlatformVideoPlayerState { } } } + } catch (e: Exception) { + if (e is CancellationException) throw e + macLogger.e { "updateFrameAsync() - Exception: ${e.message}" } } - } catch (e: Exception) { - if (e is CancellationException) throw e - macLogger.e { "updateFrameAsync() - Exception: ${e.message}" } } } @@ -854,11 +863,23 @@ class MacVideoPlayerState : PlatformVideoPlayerState { playerScope.cancel() ioScope.launch { - // Get player pointer to dispose - val ptrToDispose = mainMutex.withLock { - val ptr = playerPtr - playerPtr = null - ptr + // Get player pointer and clear cached bitmaps while frame updates are paused. + val ptrToDispose = frameMutex.withLock { + val ptrToDispose = mainMutex.withLock { + val ptr = playerPtr + playerPtr = null + ptr + } + + skiaBitmapA?.close() + skiaBitmapB?.close() + skiaBitmapA = null + skiaBitmapB = null + skiaBitmapWidth = 0 + skiaBitmapHeight = 0 + nextSkiaBitmapA = true + + ptrToDispose } // Dispose native resources outside the mutex lock @@ -874,10 +895,6 @@ class MacVideoPlayerState : PlatformVideoPlayerState { resetState() - // Clear buffered image - mainMutex.withLock { - _bufferImage = null - } } // Cancel ioScope last to ensure cleanup completes @@ -895,6 +912,7 @@ class MacVideoPlayerState : PlatformVideoPlayerState { _aspectRatio.value = 16f / 9f error = null } + lastFrameHash = Int.MIN_VALUE _currentFrameState.value = null } diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsFrameUtils.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsFrameUtils.kt new file mode 100644 index 00000000..7f80ce73 --- /dev/null +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsFrameUtils.kt @@ -0,0 +1,90 @@ +package io.github.kdroidfilter.composemediaplayer.windows + +import java.nio.ByteBuffer + +/** + * Calculates a fast hash of the frame buffer to detect frame changes. + * Samples approximately 200 pixels evenly distributed across the frame. + * + * @param buffer The source buffer containing BGRA pixel data + * @param pixelCount Total number of pixels in the frame + * @return A hash value representing the frame content + */ +internal fun calculateFrameHash(buffer: ByteBuffer, pixelCount: Int): Int { + if (pixelCount <= 0) return 0 + + var hash = 1 + val step = if (pixelCount <= 200) 1 else pixelCount / 200 + for (i in 0 until pixelCount step step) { + hash = 31 * hash + buffer.getInt(i * 4) + } + return hash +} + +/** + * Copies BGRA frame data from source to destination buffer with minimal overhead. + * Handles row padding when destination stride differs from source. + * + * This function performs a single memory copy operation when strides match, + * achieving zero-copy performance (beyond the necessary single copy from + * native buffer to Skia bitmap). + * + * @param src Source buffer containing BGRA pixel data from Media Foundation + * @param dst Destination buffer (Skia bitmap pixels via peekPixels) + * @param width Frame width in pixels + * @param height Frame height in pixels + * @param dstRowBytes Destination row stride (may include padding) + */ +internal fun copyBgraFrame( + src: ByteBuffer, + dst: ByteBuffer, + width: Int, + height: Int, + dstRowBytes: Int, +) { + require(width > 0) { "width must be > 0 (was $width)" } + require(height > 0) { "height must be > 0 (was $height)" } + val srcRowBytes = width * 4 + require(dstRowBytes >= srcRowBytes) { + "dstRowBytes ($dstRowBytes) must be >= srcRowBytes ($srcRowBytes)" + } + + val requiredSrcBytes = srcRowBytes.toLong() * height.toLong() + val requiredDstBytes = dstRowBytes.toLong() * height.toLong() + require(src.capacity().toLong() >= requiredSrcBytes) { + "src buffer too small: ${src.capacity()} < $requiredSrcBytes" + } + require(dst.capacity().toLong() >= requiredDstBytes) { + "dst buffer too small: ${dst.capacity()} < $requiredDstBytes" + } + + val srcBuf = src.duplicate() + val dstBuf = dst.duplicate() + srcBuf.rewind() + dstBuf.rewind() + + // Fast path: when strides match, do a single bulk copy + if (dstRowBytes == srcRowBytes) { + srcBuf.limit(requiredSrcBytes.toInt()) + dstBuf.limit(requiredSrcBytes.toInt()) + dstBuf.put(srcBuf) + return + } + + // Slow path: copy row by row when there's padding + val srcCapacity = srcBuf.capacity() + val dstCapacity = dstBuf.capacity() + for (row in 0 until height) { + val srcPos = row * srcRowBytes + srcBuf.limit(srcCapacity) + srcBuf.position(srcPos) + srcBuf.limit(srcPos + srcRowBytes) + + val dstPos = row * dstRowBytes + dstBuf.limit(dstCapacity) + dstBuf.position(dstPos) + dstBuf.limit(dstPos + srcRowBytes) + + dstBuf.put(srcBuf) + } +} diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt index 778b5bf2..ce6603a9 100644 --- a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt @@ -38,6 +38,7 @@ import org.jetbrains.skia.ColorAlphaType import org.jetbrains.skia.ColorType import org.jetbrains.skia.ImageInfo import java.io.File +import java.nio.ByteBuffer import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.ReentrantReadWriteLock @@ -243,7 +244,15 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { val timestamp: Double ) - // Singleton buffer to limit allocations + // Double-buffering for zero-copy frame rendering + private var skiaBitmapA: Bitmap? = null + private var skiaBitmapB: Bitmap? = null + private var nextSkiaBitmapA: Boolean = true + private var lastFrameHash: Int = Int.MIN_VALUE + private var skiaBitmapWidth: Int = 0 + private var skiaBitmapHeight: Int = 0 + + // Legacy buffer - kept for fallback but no longer used in optimized path private var sharedFrameBuffer: ByteArray? = null private var frameBitmapRecycler: Bitmap? = null @@ -351,6 +360,16 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { currentFrameState.value = null frameBitmapRecycler?.close() frameBitmapRecycler = null + + // Clean up double-buffering bitmaps + skiaBitmapA?.close() + skiaBitmapB?.close() + skiaBitmapA = null + skiaBitmapB = null + skiaBitmapWidth = 0 + skiaBitmapHeight = 0 + nextSkiaBitmapA = true + lastFrameHash = Int.MIN_VALUE } // Clear any shared buffer allocated for frames @@ -366,10 +385,10 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { isLoading = false errorMessage = null _error = null - + // Reset initialFrameRead flag to ensure we read an initial frame when reinitialized initialFrameRead.set(false) - + // Hint the GC after freeing big objects synchronously System.gc() } @@ -391,15 +410,25 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { currentFrameState.value = null frameBitmapRecycler?.close() // Recycle the bitmap if any frameBitmapRecycler = null + + // Clean up double-buffering bitmaps + skiaBitmapA?.close() + skiaBitmapB?.close() + skiaBitmapA = null + skiaBitmapB = null + skiaBitmapWidth = 0 + skiaBitmapHeight = 0 + nextSkiaBitmapA = true + lastFrameHash = Int.MIN_VALUE } // Clear any shared buffer allocated for frames sharedFrameBuffer = null frameBufferSize = 0 // Reset frame buffer size - + // Reset initialFrameRead flag to ensure we read an initial frame when reinitialized initialFrameRead.set(false) - + // Hint GC after releasing frame buffers and bitmaps System.gc() } @@ -635,6 +664,15 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { } } + /** + * Zero-copy optimized frame producer using double-buffering and direct memory access. + * + * Optimizations applied: + * 1. Double-buffering: Reuses two Bitmap objects, alternating between them. + * 2. Frame hashing: Skips processing if the frame content hasn't changed. + * 3. peekPixels(): Direct access to Skia bitmap memory, no ByteArray allocation. + * 4. Single memory copy: Native buffer → Skia bitmap pixels. + */ private suspend fun produceFrames() { while (scope.isActive && _hasMedia && !isDisposing.get()) { val instance = videoPlayerInstance ?: break @@ -644,6 +682,7 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { try { userPaused = false // Reset userPaused when looping initialFrameRead.set(false) // Reset initialFrameRead flag + lastFrameHash = Int.MIN_VALUE // Reset hash for new loop seekTo(0f) play() } catch (e: Exception) { @@ -680,66 +719,91 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { continue } - // For 4K videos, we need to be careful with memory allocation - // Only allocate a new buffer if absolutely necessary - if (sharedFrameBuffer == null) { - // First allocation - sharedFrameBuffer = ByteArray(frameBufferSize) - } else if (sharedFrameBuffer!!.size < frameBufferSize) { - // Buffer is too small, release old one before allocating new one - sharedFrameBuffer = null - System.gc() // Hint to garbage collector - delay(10) // Give GC a chance to run - sharedFrameBuffer = ByteArray(frameBufferSize) + val width = videoWidth + val height = videoHeight + + if (width <= 0 || height <= 0) { + player.UnlockVideoFrame(instance) + yield() + continue } - // Use the shared buffer with null safety check - val sharedBuffer = sharedFrameBuffer ?: run { - // Fallback if buffer is null (shouldn't happen) - ByteArray(frameBufferSize).also { sharedFrameBuffer = it } + // Get the native frame buffer + val srcBuffer = ptrRef.value.getByteBuffer(0, sizeRef.value.toLong()) + if (srcBuffer == null) { + player.UnlockVideoFrame(instance) + yield() + continue } + srcBuffer.rewind() - try { - val buffer = ptrRef.value.getByteBuffer(0, sizeRef.value.toLong()) - val copySize = min(sizeRef.value, frameBufferSize) - if (buffer != null && copySize > 0) { - buffer.get(sharedBuffer, 0, copySize) + val pixelCount = width * height + + // Calculate frame hash to detect identical frames + val newHash = calculateFrameHash(srcBuffer, pixelCount) + if (newHash == lastFrameHash) { + player.UnlockVideoFrame(instance) + yield() + continue + } + lastFrameHash = newHash + + // Reallocate bitmaps if dimensions changed + if (skiaBitmapA == null || skiaBitmapWidth != width || skiaBitmapHeight != height) { + bitmapLock.write { + skiaBitmapA?.close() + skiaBitmapB?.close() + + val imageInfo = createVideoImageInfo() + skiaBitmapA = Bitmap().apply { allocPixels(imageInfo) } + skiaBitmapB = Bitmap().apply { allocPixels(imageInfo) } + skiaBitmapWidth = width + skiaBitmapHeight = height + nextSkiaBitmapA = true } - } catch (e: Exception) { - setError("Error copying frame data: ${e.message}") - delay(100) + } + + // Select the target bitmap (double-buffering) + val targetBitmap = if (nextSkiaBitmapA) skiaBitmapA!! else skiaBitmapB!! + nextSkiaBitmapA = !nextSkiaBitmapA + + // Get direct access to bitmap pixels via peekPixels (zero-copy access) + val pixmap = targetBitmap.peekPixels() + if (pixmap == null) { + player.UnlockVideoFrame(instance) + windowsLogger.e { "Failed to get pixmap from bitmap" } + yield() + continue + } + + val pixelsAddr = pixmap.addr + if (pixelsAddr == 0L) { + player.UnlockVideoFrame(instance) + windowsLogger.e { "Invalid pixel address" } + yield() continue } + // Single memory copy: native buffer → Skia bitmap + val dstRowBytes = pixmap.rowBytes + val dstSizeBytes = dstRowBytes.toLong() * height.toLong() + val dstBuffer = Pointer(pixelsAddr).getByteBuffer(0, dstSizeBytes) + + srcBuffer.rewind() + copyBgraFrame(srcBuffer, dstBuffer, width, height, dstRowBytes) + player.UnlockVideoFrame(instance) - var bitmap = frameBitmapRecycler - if (bitmap == null) { - bitmap = Bitmap().apply { - allocPixels(createVideoImageInfo()) - } - frameBitmapRecycler = bitmap + // Get frame timestamp + val posRef = LongByReference() + val frameTime = if (player.GetMediaPosition(instance, posRef) >= 0) { + posRef.value / 10000000.0 + } else { + 0.0 } - try { - bitmap.installPixels( - createVideoImageInfo(), - sharedBuffer, - videoWidth * 4 - ) - val frameBitmap = bitmap - val posRef = LongByReference() - val frameTime = if (player.GetMediaPosition(instance, posRef) >= 0) { - posRef.value / 10000000.0 - } else { - 0.0 - } - frameChannel.trySend(FrameData(frameBitmap, frameTime)) - frameBitmapRecycler = null - } catch (e: Exception) { - windowsLogger.e { "Error processing frame bitmap: ${e.message}" } - frameBitmapRecycler = bitmap - } + // Send frame to channel + frameChannel.trySend(FrameData(targetBitmap, frameTime)) delay(1) @@ -754,6 +818,11 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { } } + /** + * Consumes frames from the channel and updates the UI. + * With zero-copy optimization, bitmaps are reused from the double-buffer pool + * and should not be closed here. + */ private suspend fun consumeFrames() { // Timeout mechanism to prevent getting stuck in loading state var frameReceived = false @@ -795,14 +864,9 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { loadingTimeout = 0 frameReceived = true + // With double-buffering, we don't close old bitmaps - they're reused + // Just update the reference and create a new ImageBitmap view bitmapLock.write { - _currentFrame?.let { oldBitmap -> - if (frameBitmapRecycler == null) { - frameBitmapRecycler = oldBitmap - } else { - oldBitmap.close() - } - } _currentFrame = frameData.bitmap // Update the currentFrameState with the new frame currentFrameState.value = frameData.bitmap.asComposeImageBitmap() @@ -951,28 +1015,17 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { if (_isPlaying) { userPaused = false } - + // Reset initialFrameRead flag to ensure we read a new frame after seeking // This is especially important if the player is paused initialFrameRead.set(false) - + + // Reset frame hash to ensure the first frame after seek is always processed + lastFrameHash = Int.MIN_VALUE + videoJob?.cancelAndJoin() clearFrameChannel() - // For 4K videos, we need to be careful with memory allocation - // Only allocate a new buffer if absolutely necessary - if (sharedFrameBuffer == null) { - // First allocation - sharedFrameBuffer = ByteArray(frameBufferSize) - } else if (sharedFrameBuffer!!.size < frameBufferSize) { - // Buffer is too small, release old one before allocating new one - sharedFrameBuffer = null - System.gc() // Hint to garbage collector - delay(10) // Give GC a chance to run - sharedFrameBuffer = ByteArray(frameBufferSize) - } - // If buffer exists and is large enough, reuse it - val targetPos = (_duration * (value / 1000f) * 10000000).toLong() var hr = player.SeekMedia(instance, targetPos) if (hr < 0) { diff --git a/mediaplayer/src/jvmTest/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtilsTest.kt b/mediaplayer/src/jvmTest/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtilsTest.kt new file mode 100644 index 00000000..fd059895 --- /dev/null +++ b/mediaplayer/src/jvmTest/kotlin/io/github/kdroidfilter/composemediaplayer/mac/MacFrameUtilsTest.kt @@ -0,0 +1,113 @@ +package io.github.kdroidfilter.composemediaplayer.mac + +import java.nio.ByteBuffer +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotEquals + +class MacFrameUtilsTest { + @Test + fun calculateFrameHash_returnsZeroWhenEmpty() { + assertEquals(0, calculateFrameHash(ByteBuffer.allocate(0), 0)) + assertEquals(0, calculateFrameHash(ByteBuffer.allocate(0), -1)) + } + + @Test + fun calculateFrameHash_changesWhenSampledPixelChanges() { + val pixelCount = 1000 + val buf = ByteBuffer.allocate(pixelCount * 4) + for (i in 0 until pixelCount) { + buf.putInt(i * 4, i) + } + + val hash1 = calculateFrameHash(buf, pixelCount) + + // With pixelCount=1000, step=5 => index 5 is sampled. + buf.putInt(5 * 4, 123456) + val hash2 = calculateFrameHash(buf, pixelCount) + + assertNotEquals(hash1, hash2) + } + + @Test + fun copyBgraFrame_copiesContiguousRows() { + val width = 2 + val height = 2 + val rowBytes = width * 4 + val size = rowBytes * height + + val src = ByteBuffer.allocate(size) + for (i in 0 until size) { + src.put(i, i.toByte()) + } + val dst = ByteBuffer.allocate(size) + + copyBgraFrame(src, dst, width, height, rowBytes) + + for (i in 0 until size) { + assertEquals(src.get(i), dst.get(i), "Mismatch at byte index $i") + } + } + + @Test + fun copyBgraFrame_copiesWithRowPadding() { + val width = 2 + val height = 2 + val srcRowBytes = width * 4 + val dstRowBytes = srcRowBytes + 4 + + val srcSize = srcRowBytes * height + val dstSize = dstRowBytes * height + + val src = ByteBuffer.allocate(srcSize) + for (i in 0 until srcSize) { + src.put(i, (i + 1).toByte()) + } + + val dst = ByteBuffer.allocate(dstSize) + val paddingSentinel = 0x7F.toByte() + for (i in 0 until dstSize) { + dst.put(i, paddingSentinel) + } + + copyBgraFrame(src, dst, width, height, dstRowBytes) + + for (row in 0 until height) { + val srcBase = row * srcRowBytes + val dstBase = row * dstRowBytes + + for (i in 0 until srcRowBytes) { + assertEquals( + src.get(srcBase + i), + dst.get(dstBase + i), + "Row $row mismatch at byte index $i", + ) + } + + for (i in srcRowBytes until dstRowBytes) { + assertEquals( + paddingSentinel, + dst.get(dstBase + i), + "Row $row padding byte $i should be untouched", + ) + } + } + } + + @Test + fun copyBgraFrame_requiresValidRowBytes() { + val width = 2 + val height = 1 + val srcRowBytes = width * 4 + val dstRowBytes = srcRowBytes - 1 + + val src = ByteBuffer.allocate(srcRowBytes * height) + val dst = ByteBuffer.allocate(dstRowBytes * height) + + assertFailsWith { + copyBgraFrame(src, dst, width, height, dstRowBytes) + } + } +} + From a86e4c5787d9250e76c62be47d66bb867f5c0d8a Mon Sep 17 00:00:00 2001 From: "Elie G." Date: Tue, 23 Dec 2025 21:09:50 +0200 Subject: [PATCH 2/2] Refine bitmap cleanup in `WindowsVideoPlayerState` to avoid unintended disposal of shared frames. --- .../windows/WindowsVideoPlayerState.kt | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt index ce6603a9..3ebb12f3 100644 --- a/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt +++ b/mediaplayer/src/jvmMain/kotlin/io/github/kdroidfilter/composemediaplayer/windows/WindowsVideoPlayerState.kt @@ -355,7 +355,14 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { // Free bitmaps and frame buffers bitmapLock.write { - _currentFrame?.close() + val currentFrame = _currentFrame + if (currentFrame != null && + currentFrame !== skiaBitmapA && + currentFrame !== skiaBitmapB && + currentFrame !== frameBitmapRecycler + ) { + currentFrame.close() + } _currentFrame = null currentFrameState.value = null frameBitmapRecycler?.close() @@ -404,7 +411,14 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { // Free bitmaps and frame buffers bitmapLock.write { - _currentFrame?.close() // Close the current frame bitmap if any + val currentFrame = _currentFrame + if (currentFrame != null && + currentFrame !== skiaBitmapA && + currentFrame !== skiaBitmapB && + currentFrame !== frameBitmapRecycler + ) { + currentFrame.close() + } _currentFrame = null // Reset the currentFrameState currentFrameState.value = null @@ -1284,4 +1298,4 @@ class WindowsVideoPlayerState : PlatformVideoPlayerState { override fun toggleFullscreen() { isFullscreen = !isFullscreen } -} \ No newline at end of file +}