diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index d58e1b791870e..67c5e390185eb 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -121,10 +121,20 @@ final class SurfaceTextureRegistryEntry implements TextureRegistry.SurfaceTextur private final long id; @NonNull private final SurfaceTextureWrapper textureWrapper; private boolean released; + @Nullable private OnFrameConsumedListener listener; + private final Runnable onFrameConsumed = + new Runnable() { + @Override + public void run() { + if (listener != null) { + listener.onFrameConsumed(); + } + } + }; SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) { this.id = id; - this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture); + this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { // The callback relies on being executed on the UI thread (unsynchronised read of @@ -195,6 +205,11 @@ protected void finalize() throws Throwable { super.finalize(); } } + + @Override + public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) { + this.listener = listener; + } } static final class SurfaceTextureFinalizerRunnable implements Runnable { diff --git a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java index 379fd3c558fc1..655d0db865b4e 100644 --- a/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java +++ b/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java @@ -7,6 +7,7 @@ import android.graphics.SurfaceTexture; import androidx.annotation.Keep; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; /** * A wrapper for a SurfaceTexture that tracks whether the texture has been released. @@ -20,10 +21,25 @@ public class SurfaceTextureWrapper { private SurfaceTexture surfaceTexture; private boolean released; private boolean attached; + private Runnable onFrameConsumed; public SurfaceTextureWrapper(@NonNull SurfaceTexture surfaceTexture) { + this(surfaceTexture, null); + } + + /** + * A wrapper for a SurfaceTexture. + * + *

The provided {@code onFrameConsumed} callback must be invoked when the most recent image was + * consumed. + * + * @param onFrameConsumed The callback after the {@code updateTexImage} is called. + */ + public SurfaceTextureWrapper( + @NonNull SurfaceTexture surfaceTexture, @Nullable Runnable onFrameConsumed) { this.surfaceTexture = surfaceTexture; this.released = false; + this.onFrameConsumed = onFrameConsumed; } @NonNull @@ -37,6 +53,9 @@ public void updateTexImage() { synchronized (this) { if (!released) { surfaceTexture.updateTexImage(); + if (onFrameConsumed != null) { + onFrameConsumed.run(); + } } } } diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java index ba0205a7b76df..4cbc514cb7ced 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java @@ -26,6 +26,8 @@ import io.flutter.Log; import io.flutter.embedding.android.AndroidTouchProcessor; import io.flutter.util.ViewUtils; +import io.flutter.view.TextureRegistry; +import java.util.concurrent.atomic.AtomicLong; /** * Wraps a platform view to intercept gestures and project this view onto a {@link SurfaceTexture}. @@ -52,12 +54,45 @@ class PlatformViewWrapper extends FrameLayout { private AndroidTouchProcessor touchProcessor; @Nullable @VisibleForTesting ViewTreeObserver.OnGlobalFocusChangeListener activeFocusListener; + @Nullable private TextureRegistry.SurfaceTextureEntry textureEntry; + private final AtomicLong pendingFramesCount = new AtomicLong(0L); + + private final TextureRegistry.OnFrameConsumedListener listener = + new TextureRegistry.OnFrameConsumedListener() { + @Override + public void onFrameConsumed() { + if (Build.VERSION.SDK_INT == 29) { + pendingFramesCount.decrementAndGet(); + } + } + }; + + private void onFrameProduced() { + if (Build.VERSION.SDK_INT == 29) { + pendingFramesCount.incrementAndGet(); + } + } + + private boolean shouldDrawToSurfaceNow() { + if (Build.VERSION.SDK_INT == 29) { + return pendingFramesCount.get() <= 0L; + } + return true; + } public PlatformViewWrapper(@NonNull Context context) { super(context); setWillNotDraw(false); } + public PlatformViewWrapper( + @NonNull Context context, @NonNull TextureRegistry.SurfaceTextureEntry textureEntry) { + this(context); + this.textureEntry = textureEntry; + textureEntry.setOnFrameConsumedListener(listener); + setTexture(textureEntry.surfaceTexture()); + } + /** * Sets the touch processor that allows to intercept gestures. * @@ -109,6 +144,7 @@ public void setTexture(@Nullable SurfaceTexture newTx) { } else { canvas.drawColor(Color.TRANSPARENT); } + onFrameProduced(); } finally { surface.unlockCanvasAndPost(canvas); } @@ -202,19 +238,29 @@ public void draw(Canvas canvas) { Log.e(TAG, "Invalid texture. The platform view cannot be displayed."); return; } - // Override the canvas that this subtree of views will use to draw. - final Canvas surfaceCanvas = surface.lockHardwareCanvas(); - try { - // Clear the current pixels in the canvas. - // This helps when a WebView renders an HTML document with transparent background. - if (Build.VERSION.SDK_INT >= 29) { - surfaceCanvas.drawColor(Color.TRANSPARENT, BlendMode.CLEAR); - } else { - surfaceCanvas.drawColor(Color.TRANSPARENT); + // We've observed on Android Q that we have to wait for the consumer of {@link SurfaceTexture} + // to consume the last image before continuing to draw, otherwise subsequent calls to + // {@code dequeueBuffer} to request a free buffer from the {@link BufferQueue} will fail. + // See https://github.com/flutter/flutter/issues/98722 + if (!shouldDrawToSurfaceNow()) { + // If there are still frames that are not consumed, we will draw them next time. + invalidate(); + } else { + // Override the canvas that this subtree of views will use to draw. + final Canvas surfaceCanvas = surface.lockHardwareCanvas(); + try { + // Clear the current pixels in the canvas. + // This helps when a WebView renders an HTML document with transparent background. + if (Build.VERSION.SDK_INT >= 29) { + surfaceCanvas.drawColor(Color.TRANSPARENT, BlendMode.CLEAR); + } else { + surfaceCanvas.drawColor(Color.TRANSPARENT); + } + super.draw(surfaceCanvas); + onFrameProduced(); + } finally { + surface.unlockCanvasAndPost(surfaceCanvas); } - super.draw(surfaceCanvas); - } finally { - surface.unlockCanvasAndPost(surfaceCanvas); } } diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index 3d9e86d749557..c8326a387e777 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -188,10 +188,9 @@ public long createForTextureLayer( final PlatformView platformView = viewFactory.create(context, viewId, createParams); platformViews.put(viewId, platformView); - final PlatformViewWrapper wrapperView = new PlatformViewWrapper(context); final TextureRegistry.SurfaceTextureEntry textureEntry = textureRegistry.createSurfaceTexture(); - wrapperView.setTexture(textureEntry.surfaceTexture()); + final PlatformViewWrapper wrapperView = new PlatformViewWrapper(context, textureEntry); wrapperView.setTouchProcessor(androidTouchProcessor); final int physicalWidth = toPhysicalPixels(request.logicalWidth); diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java index f47b8da5bde00..0db6344997802 100644 --- a/shell/platform/android/io/flutter/view/FlutterView.java +++ b/shell/platform/android/io/flutter/view/FlutterView.java @@ -917,6 +917,7 @@ public void onFrameAvailable(SurfaceTexture texture) { // still be called by a stale reference after released==true and mNativeView==null. return; } + mNativeView .getFlutterJNI() .markTextureFrameAvailable(SurfaceTextureRegistryEntry.this.id); diff --git a/shell/platform/android/io/flutter/view/TextureRegistry.java b/shell/platform/android/io/flutter/view/TextureRegistry.java index 1155c4854bf66..ed890f0d38b3f 100644 --- a/shell/platform/android/io/flutter/view/TextureRegistry.java +++ b/shell/platform/android/io/flutter/view/TextureRegistry.java @@ -6,6 +6,7 @@ import android.graphics.SurfaceTexture; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; // TODO(mattcarroll): re-evalute docs in this class and add nullability annotations. /** @@ -41,5 +42,17 @@ interface SurfaceTextureEntry { /** Deregisters and releases this SurfaceTexture. */ void release(); + + /** Set a listener that will be notified when the most recent image has been consumed. */ + default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) {} + } + + /** Listener invoked when the most recent image has been consumed. */ + interface OnFrameConsumedListener { + /** + * This method will to be invoked when the most recent image from the image stream has been + * consumed. + */ + void onFrameConsumed(); } } diff --git a/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java b/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java index e38001a7231c2..110a8c9bb5ee8 100644 --- a/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java +++ b/shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java @@ -18,8 +18,10 @@ import android.view.Surface; import androidx.test.ext.junit.runners.AndroidJUnit4; import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.view.TextureRegistry; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -312,4 +314,29 @@ public void itConvertsDisplayFeatureArrayToPrimitiveArrays() { }, stateCaptor.getValue()); } + + @Test + public void itNotifyImageFrameListener() { + // Setup the test. + FlutterRenderer flutterRenderer = new FlutterRenderer(fakeFlutterJNI); + + AtomicInteger invocationCount = new AtomicInteger(0); + final TextureRegistry.OnFrameConsumedListener listener = + new TextureRegistry.OnFrameConsumedListener() { + @Override + public void onFrameConsumed() { + invocationCount.incrementAndGet(); + } + }; + + FlutterRenderer.SurfaceTextureRegistryEntry entry = + (FlutterRenderer.SurfaceTextureRegistryEntry) flutterRenderer.createSurfaceTexture(); + entry.setOnFrameConsumedListener(listener); + + // Execute the behavior under test. + entry.textureWrapper().updateTexImage(); + + // Verify behavior under test. + assertEquals(1, invocationCount.get()); + } }