Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,20 @@ final class SurfaceTextureRegistryEntry implements TextureRegistry.SurfaceTextur
private final long id;
@NonNull private final SurfaceTextureWrapper textureWrapper;
private boolean released;
@Nullable private OnFrameConsumedListener listener;
private final Runnable onFrameConsumed =
new Runnable() {
@Override
public void run() {
if (listener != null) {
listener.onFrameConsumed();
}
}
};

SurfaceTextureRegistryEntry(long id, @NonNull SurfaceTexture surfaceTexture) {
this.id = id;
this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture);
this.textureWrapper = new SurfaceTextureWrapper(surfaceTexture, onFrameConsumed);

if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
// The callback relies on being executed on the UI thread (unsynchronised read of
Expand Down Expand Up @@ -195,6 +205,11 @@ protected void finalize() throws Throwable {
super.finalize();
}
}

@Override
public void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) {
this.listener = listener;
}
}

static final class SurfaceTextureFinalizerRunnable implements Runnable {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import android.graphics.SurfaceTexture;
import androidx.annotation.Keep;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;

/**
* A wrapper for a SurfaceTexture that tracks whether the texture has been released.
Expand All @@ -20,10 +21,25 @@ public class SurfaceTextureWrapper {
private SurfaceTexture surfaceTexture;
private boolean released;
private boolean attached;
private Runnable onFrameConsumed;

public SurfaceTextureWrapper(@NonNull SurfaceTexture surfaceTexture) {
this(surfaceTexture, null);
}

/**
* A wrapper for a SurfaceTexture.
*
* <p>The provided {@code onFrameConsumed} callback must be invoked when the most recent image was
* consumed.
*
* @param onFrameConsumed The callback after the {@code updateTexImage} is called.
*/
public SurfaceTextureWrapper(
@NonNull SurfaceTexture surfaceTexture, @Nullable Runnable onFrameConsumed) {
Comment on lines +38 to +39
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

java doc for this constructor as well as the previous one to document when to use which

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done.

this.surfaceTexture = surfaceTexture;
this.released = false;
this.onFrameConsumed = onFrameConsumed;
}

@NonNull
Expand All @@ -37,6 +53,9 @@ public void updateTexImage() {
synchronized (this) {
if (!released) {
surfaceTexture.updateTexImage();
if (onFrameConsumed != null) {
onFrameConsumed.run();
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import io.flutter.Log;
import io.flutter.embedding.android.AndroidTouchProcessor;
import io.flutter.util.ViewUtils;
import io.flutter.view.TextureRegistry;
import java.util.concurrent.atomic.AtomicLong;

/**
* Wraps a platform view to intercept gestures and project this view onto a {@link SurfaceTexture}.
Expand All @@ -52,12 +54,45 @@ class PlatformViewWrapper extends FrameLayout {
private AndroidTouchProcessor touchProcessor;

@Nullable @VisibleForTesting ViewTreeObserver.OnGlobalFocusChangeListener activeFocusListener;
@Nullable private TextureRegistry.SurfaceTextureEntry textureEntry;
private final AtomicLong pendingFramesCount = new AtomicLong(0L);

private final TextureRegistry.OnFrameConsumedListener listener =
new TextureRegistry.OnFrameConsumedListener() {
@Override
public void onFrameConsumed() {
if (Build.VERSION.SDK_INT == 29) {
pendingFramesCount.decrementAndGet();
}
}
};

private void onFrameProduced() {
if (Build.VERSION.SDK_INT == 29) {
pendingFramesCount.incrementAndGet();
}
}

private boolean shouldDrawToSurfaceNow() {
if (Build.VERSION.SDK_INT == 29) {
return pendingFramesCount.get() <= 0L;
}
return true;
}

public PlatformViewWrapper(@NonNull Context context) {
super(context);
setWillNotDraw(false);
}

public PlatformViewWrapper(
@NonNull Context context, @NonNull TextureRegistry.SurfaceTextureEntry textureEntry) {
this(context);
this.textureEntry = textureEntry;
textureEntry.setOnFrameConsumedListener(listener);
setTexture(textureEntry.surfaceTexture());
}

/**
* Sets the touch processor that allows to intercept gestures.
*
Expand Down Expand Up @@ -109,6 +144,7 @@ public void setTexture(@Nullable SurfaceTexture newTx) {
} else {
canvas.drawColor(Color.TRANSPARENT);
}
onFrameProduced();
} finally {
surface.unlockCanvasAndPost(canvas);
}
Expand Down Expand Up @@ -202,19 +238,29 @@ public void draw(Canvas canvas) {
Log.e(TAG, "Invalid texture. The platform view cannot be displayed.");
return;
}
// Override the canvas that this subtree of views will use to draw.
final Canvas surfaceCanvas = surface.lockHardwareCanvas();
try {
// Clear the current pixels in the canvas.
// This helps when a WebView renders an HTML document with transparent background.
if (Build.VERSION.SDK_INT >= 29) {
surfaceCanvas.drawColor(Color.TRANSPARENT, BlendMode.CLEAR);
} else {
surfaceCanvas.drawColor(Color.TRANSPARENT);
// We've observed on Android Q that we have to wait for the consumer of {@link SurfaceTexture}
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

did you test with versions below Q?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @jreck

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

did you test with versions below Q?

Yes, I did. They work fine.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to clarify, which versions did you test on? I'm getting reports that it may be several versions. See googleads/googleads-mobile-flutter#269 (comment)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to clarify, which versions did you test on? I'm getting reports that it may be several versions. See googleads/googleads-mobile-flutter#269 (comment)

I have tested on real devices from Android9 to Android12.

// to consume the last image before continuing to draw, otherwise subsequent calls to
// {@code dequeueBuffer} to request a free buffer from the {@link BufferQueue} will fail.
// See https://github.com/flutter/flutter/issues/98722
if (!shouldDrawToSurfaceNow()) {
// If there are still frames that are not consumed, we will draw them next time.
invalidate();
} else {
// Override the canvas that this subtree of views will use to draw.
final Canvas surfaceCanvas = surface.lockHardwareCanvas();
try {
// Clear the current pixels in the canvas.
// This helps when a WebView renders an HTML document with transparent background.
if (Build.VERSION.SDK_INT >= 29) {
surfaceCanvas.drawColor(Color.TRANSPARENT, BlendMode.CLEAR);
} else {
surfaceCanvas.drawColor(Color.TRANSPARENT);
}
super.draw(surfaceCanvas);
onFrameProduced();
} finally {
surface.unlockCanvasAndPost(surfaceCanvas);
}
super.draw(surfaceCanvas);
} finally {
surface.unlockCanvasAndPost(surfaceCanvas);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,10 +188,9 @@ public long createForTextureLayer(
final PlatformView platformView = viewFactory.create(context, viewId, createParams);
platformViews.put(viewId, platformView);

final PlatformViewWrapper wrapperView = new PlatformViewWrapper(context);
final TextureRegistry.SurfaceTextureEntry textureEntry =
textureRegistry.createSurfaceTexture();
wrapperView.setTexture(textureEntry.surfaceTexture());
final PlatformViewWrapper wrapperView = new PlatformViewWrapper(context, textureEntry);
wrapperView.setTouchProcessor(androidTouchProcessor);

final int physicalWidth = toPhysicalPixels(request.logicalWidth);
Expand Down
1 change: 1 addition & 0 deletions shell/platform/android/io/flutter/view/FlutterView.java
Original file line number Diff line number Diff line change
Expand Up @@ -917,6 +917,7 @@ public void onFrameAvailable(SurfaceTexture texture) {
// still be called by a stale reference after released==true and mNativeView==null.
return;
}

mNativeView
.getFlutterJNI()
.markTextureFrameAvailable(SurfaceTextureRegistryEntry.this.id);
Expand Down
13 changes: 13 additions & 0 deletions shell/platform/android/io/flutter/view/TextureRegistry.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import android.graphics.SurfaceTexture;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;

// TODO(mattcarroll): re-evalute docs in this class and add nullability annotations.
/**
Expand Down Expand Up @@ -41,5 +42,17 @@ interface SurfaceTextureEntry {

/** Deregisters and releases this SurfaceTexture. */
void release();

/** Set a listener that will be notified when the most recent image has been consumed. */
default void setOnFrameConsumedListener(@Nullable OnFrameConsumedListener listener) {}
}

/** Listener invoked when the most recent image has been consumed. */
interface OnFrameConsumedListener {
/**
* This method will to be invoked when the most recent image from the image stream has been
* consumed.
*/
void onFrameConsumed();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
import android.view.Surface;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import io.flutter.embedding.engine.FlutterJNI;
import io.flutter.view.TextureRegistry;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down Expand Up @@ -312,4 +314,29 @@ public void itConvertsDisplayFeatureArrayToPrimitiveArrays() {
},
stateCaptor.getValue());
}

@Test
public void itNotifyImageFrameListener() {
// Setup the test.
FlutterRenderer flutterRenderer = new FlutterRenderer(fakeFlutterJNI);

AtomicInteger invocationCount = new AtomicInteger(0);
final TextureRegistry.OnFrameConsumedListener listener =
new TextureRegistry.OnFrameConsumedListener() {
@Override
public void onFrameConsumed() {
invocationCount.incrementAndGet();
}
};

FlutterRenderer.SurfaceTextureRegistryEntry entry =
(FlutterRenderer.SurfaceTextureRegistryEntry) flutterRenderer.createSurfaceTexture();
entry.setOnFrameConsumedListener(listener);

// Execute the behavior under test.
entry.textureWrapper().updateTexImage();

// Verify behavior under test.
assertEquals(1, invocationCount.get());
}
}