Fix a "black screen flash" which occurs sometimes when restarting a track

.changeset/open-candles-talk.md

@@ -0,0 +1,5 @@
+---
+'@livekit/track-processors': patch
+---
+
+Fix a "black screen flash" which occurs sometimes when restarting the processor wrapper

example/index.html

@@ -96,16 +96,25 @@ <h2>LiveKit track processor sample</h2>
               </button>
 
               <div class="card mt-1 mb-1 mx-n2 p-2">
-                <div>
+                <div class="d-flex align-items-center justify-content-between">
                   <button
                     id="toggle-track-processor"
-                    class="btn btn-secondary"
+                    class="btn btn-secondary mr-2"
                     disabled
                     type="button"
                     onclick="appActions.toggleTrackProcessorEnabled()"
                   >
                     Insert Track Processor
                   </button>
+
+                  <div id="initial-mode-wrapper" style="display: none;">
+                    <span class="text-secondary">Initial mode:</span>
+                    <select class="custom-select" id="initial-mode-select" style="width: 200px;">
+                      <option value="disabled">Disabled</option>
+                      <option value="background-blur">Blur</option>
+                      <option value="virtual-background">Virtual Background</option>
+                    </select>
+                  </div>
                 </div>
 
                 <div id="track-processor-modes" style="display: none;">

example/sample.ts

@@ -254,8 +254,23 @@ const appActions = {
       if (state.isBackgroundProcessorEnabled) {
         await camTrack.stopProcessor();
         state.isBackgroundProcessorEnabled = false;
+        $("initial-mode-wrapper").style.display = 'block';
       } else {
-        await state.backgroundProcessor.switchTo({ mode: 'disabled' });
+        $("initial-mode-wrapper").style.display = 'none';
+        const initialMode = $<HTMLSelectElement>("initial-mode-select").value as BackgroundProcessorOptions['mode'];
+
+        switch (initialMode) {
+          case 'disabled':
+            await state.backgroundProcessor.switchTo({ mode: 'disabled' });
+            break;
+          case 'virtual-background':
+            await state.backgroundProcessor.switchTo({ mode: 'virtual-background', imagePath: IMAGE_PATH });
+            break;
+          case 'background-blur':
+            await state.backgroundProcessor.switchTo({ mode: 'background-blur' });
+            break;
+        }
+
         state.isBackgroundProcessorEnabled = true;
         await camTrack.setProcessor(state.backgroundProcessor);
       }
@@ -655,6 +670,8 @@ function setButtonsForState(connected: boolean) {
 
   toRemove.forEach((id) => $(id)?.removeAttribute('disabled'));
   toAdd.forEach((id) => $(id)?.setAttribute('disabled', 'true'));
+
+  $("initial-mode-wrapper").style.display = connected ? 'block' : 'none';
 }
 
 const elementMapping: { [k: string]: MediaDeviceKind } = {
@@ -716,11 +733,12 @@ function updateButtonsForPublishState() {
 }
 
 function updateTrackProcessorModeButtons() {
+  const toggleTrackProcessorButtonEnabled = currentRoom?.state === ConnectionState.Connected;
   if (state.isBackgroundProcessorEnabled) {
-    setButtonState('toggle-track-processor', 'Remove Track Processor', false, false);
+    setButtonState('toggle-track-processor', 'Remove Track Processor', false, !toggleTrackProcessorButtonEnabled);
     $('track-processor-modes').style.display = 'block';
   } else {
-    setButtonState('toggle-track-processor', 'Insert Track Processor', false, false);
+    setButtonState('toggle-track-processor', 'Insert Track Processor', false, !toggleTrackProcessorButtonEnabled);
     $('track-processor-modes').style.display = 'none';
   }
 

src/ProcessorWrapper.ts

@@ -1,8 +1,10 @@
 import { type ProcessorOptions, type Track, type TrackProcessor } from 'livekit-client';
-import { TrackTransformer } from './transformers';
+import { TrackTransformer, TrackTransformerDestroyOptions } from './transformers';
 import { createCanvas, waitForTrackResolution } from './utils';
 import { LoggerNames, getLogger } from './logger';
 
+type ProcessorWrapperLifecycleState = 'idle' | 'initializing' | 'running' | 'media-exhausted' | 'destroying' | 'destroyed';
+
 export interface ProcessorWrapperOptions {
   /**
    * Maximum frame rate for fallback canvas.captureStream implementation
@@ -81,10 +83,10 @@ export default class ProcessorWrapper<
   // FPS control for fallback implementation
   private maxFps: number;
 
-  private symbol?: Symbol;
-
   private log = getLogger(LoggerNames.ProcessorWrapper);
 
+  private lifecycleState: ProcessorWrapperLifecycleState = 'idle';
+
   constructor(
     transformer: Transformer,
     name: string,
@@ -146,6 +148,8 @@ export default class ProcessorWrapper<
   }
 
   async init(opts: ProcessorOptions<Track.Kind>): Promise<void> {
+    this.log.debug('Init called');
+    this.lifecycleState = 'initializing';
     await this.setup(opts);
 
     if (!this.canvas) {
@@ -162,6 +166,7 @@ export default class ProcessorWrapper<
     } else {
       this.initStreamProcessorPath();
     }
+    this.lifecycleState = 'running';
   }
 
   private initStreamProcessorPath() {
@@ -174,20 +179,20 @@ export default class ProcessorWrapper<
     const readableStream = this.processor.readable;
     const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);
 
-    const symbol = Symbol('stream');
-    this.symbol = symbol;
-
     pipedStream
       .pipeTo(this.trackGenerator.writable)
-      // destroy processor if stream finishes
-      .then(() => this.destroy(symbol))
+      // if stream finishes, the media to process is exhausted
+      .then(() => this.handleMediaExhausted())
       // destroy processor if stream errors - unless it's an abort error
       .catch((e) => {
         if (e instanceof DOMException && e.name === 'AbortError') {
           this.log.log('stream processor path aborted');
+        } else if (e instanceof DOMException && e.name === 'InvalidStateError' && e.message === 'Stream closed') {
+          this.log.log('stream processor underlying stream closed');
+          this.handleMediaExhausted();
         } else {
           this.log.error('error when trying to pipe', e);
-          this.destroy(symbol);
+          this.destroy();
         }
       });
 
@@ -241,6 +246,7 @@ export default class ProcessorWrapper<
 
   private startRenderLoop() {
     if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
+      this.handleMediaExhausted();
       return;
     }
 
@@ -263,6 +269,7 @@ export default class ProcessorWrapper<
         !this.sourceDummy ||
         !(this.sourceDummy instanceof HTMLVideoElement)
       ) {
+        this.handleMediaExhausted();
         return;
       }
 
@@ -349,7 +356,8 @@ export default class ProcessorWrapper<
   }
 
   async restart(opts: ProcessorOptions<Track.Kind>): Promise<void> {
-    await this.destroy();
+    this.log.debug('Restart called');
+    await this.destroy({ willProcessorRestart: true });
     await this.init(opts);
   }
 
@@ -362,11 +370,18 @@ export default class ProcessorWrapper<
     await this.transformer.update(options[0]);
   }
 
-  async destroy(symbol?: Symbol) {
-    if (symbol && this.symbol !== symbol) {
-      // If the symbol is provided, we only destroy if it matches the current symbol
+  /** Called if the media pipeline no longer can read frames to process from the source media */
+  private async handleMediaExhausted() {
+    this.log.debug('Media was exhausted from source');
+    if (this.lifecycleState !== 'running') {
       return;
     }
+    this.lifecycleState = 'media-exhausted'
+    await this.cleanup();
+  }
+
+  /** Tears down the media stack logic initialized in initStreamProcessorPath / initFallbackPath */
+  private async cleanup() {
     if (this.useStreamFallback) {
       this.processingEnabled = false;
       if (this.animationFrameId) {
@@ -378,9 +393,28 @@ export default class ProcessorWrapper<
       }
       this.capturedStream?.getTracks().forEach((track) => track.stop());
     } else {
+      // NOTE: closing writableControl below terminates the stream in initStreamProcessorPath /
+      // calls the .then(...) which calls this.handleMediaExhausted
       await this.processor?.writableControl?.close();
       this.trackGenerator?.stop();
     }
-    await this.transformer.destroy();
+  }
+
+  async destroy(transformerDestroyOptions: TrackTransformerDestroyOptions = { willProcessorRestart: false }) {
+    this.log.debug(`Destroy called - lifecycleState=${this.lifecycleState}, transformerDestroyOptions=${JSON.stringify(transformerDestroyOptions)}`);
+    switch (this.lifecycleState) {
+      case 'running':
+      case 'media-exhausted':
+        this.lifecycleState = 'destroying';
+
+        await this.cleanup();
+
+        await this.transformer.destroy(transformerDestroyOptions);
+        this.lifecycleState = 'destroyed';
+        break;
+
+      default:
+        break;
+    }
   }
 }

src/transformers/BackgroundTransformer.ts

@@ -2,7 +2,7 @@ import * as vision from '@mediapipe/tasks-vision';
 import { getLogger, LoggerNames } from '../logger';
 import { dependencies } from '../../package.json';
 import VideoTransformer from './VideoTransformer';
-import { VideoTransformerInitOptions } from './types';
+import { TrackTransformerDestroyOptions, VideoTransformerInitOptions } from './types';
 
 export type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;
 
@@ -90,11 +90,14 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
     this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
   }
 
-  async destroy() {
+  async destroy(options?: TrackTransformerDestroyOptions) {
     await super.destroy();
     await this.imageSegmenter?.close();
     this.backgroundImageAndPath = null;
-    this.isFirstFrame = true;
+
+    if (!options?.willProcessorRestart) {
+      this.isFirstFrame = true;
+    }
   }
 
   async loadAndSetBackground(path: string) {

src/transformers/types.ts

@@ -10,36 +10,39 @@ export interface VideoTransformerInitOptions extends TrackTransformerInitOptions
 export interface AudioTransformerInitOptions extends TrackTransformerInitOptions {}
 
 export interface VideoTrackTransformer<Options extends Record<string, unknown>>
-  extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame> {
+  extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame, TrackTransformerDestroyOptions> {
   init: (options: VideoTransformerInitOptions) => void;
-  destroy: () => void;
+  destroy: (options?: TrackTransformerDestroyOptions) => void;
   restart: (options: VideoTransformerInitOptions) => void;
   transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
   update: (options: Options) => void;
 }
 
 export interface AudioTrackTransformer<Options extends Record<string, unknown>>
-  extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData> {
+  extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData, TrackTransformerDestroyOptions> {
   init: (options: AudioTransformerInitOptions) => void;
-  destroy: () => void;
+  destroy: (options: TrackTransformerDestroyOptions) => void;
   restart: (options: AudioTransformerInitOptions) => void;
   transform: (frame: AudioData, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
   update: (options: Options) => void;
 }
 
+export type TrackTransformerDestroyOptions = { willProcessorRestart: boolean };
+
 export type TrackTransformer<Options extends Record<string, unknown>> =
   | VideoTrackTransformer<Options>
   | AudioTrackTransformer<Options>;
 
 export interface BaseTrackTransformer<
-  T extends TrackTransformerInitOptions,
+  InitOpts extends TrackTransformerInitOptions,
   DataType extends VideoFrame | AudioData,
+  DestroyOpts extends TrackTransformerDestroyOptions = TrackTransformerDestroyOptions,
 > {
-  init: (options: T) => void;
-  destroy: () => void;
-  restart: (options: T) => void;
+  init: (options: InitOpts) => void;
+  destroy: (options: DestroyOpts) => void;
+  restart: (options: InitOpts) => void;
   transform: (frame: DataType, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
 }