Cwilson/audio processor

.changeset/add-audio-processor.md

@@ -0,0 +1,5 @@
+---
+"@livekit/track-processors": minor
+---
+
+Add GainAudioProcessor — a reference audio TrackProcessor implementation using the Web Audio API. Includes gain control, browser support detection via `isSupported`, and a complete example in the sample app. Also restructures documentation into separate video and audio processor guides.

README.md

@@ -1,91 +1,51 @@
 # LiveKit track processors
 
+Prebuilt audio and video track processors for [LiveKit](https://livekit.io), implementing the [`TrackProcessor`](https://docs.livekit.io/reference/client-sdk-js/interfaces/TrackProcessor.html) interface from `livekit-client`.
+
 ## Install
 
 ```
 npm add @livekit/track-processors
 ```
 
-## Usage of prebuilt processors
-
-### Available processors
-
-This package exposes the `BackgroundProcessor` pre-prepared processor pipeline, which can be used in a few ways:
+## Video processors
 
-- `BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 /* (optional) */ })`
-- `BackgroundProcessor({ mode: 'virtual-background', imagePath: "http://path.to/image.png" })`
-- `BackgroundProcessor({ mode: 'disabled' })`
-
-### Usage example
+Video track processors intercept a local video track's frames and transform them before they are sent to other participants. This package provides a prebuilt `BackgroundProcessor` that supports background blur and virtual backgrounds:
 
 ```ts
-import { BackgroundProcessor, supportsBackgroundProcessors, supportsModernBackgroundProcessors } from '@livekit/track-processors';
-
-if(!supportsBackgroundProcessors()) {
-  throw new Error("this browser does not support background processors")
-}
+import { BackgroundProcessor } from '@livekit/track-processors';
 
-if(supportsModernBackgroundProcessors()) {
-  console.log("this browser supports modern APIs that are more performant");
-}
-
-const videoTrack = await createLocalVideoTrack();
-const processor = BackgroundProcessor({ mode: 'background-blur' });
+const processor = BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 });
 await videoTrack.setProcessor(processor);
-room.localParticipant.publishTrack(videoTrack);
-
-async function disableBackgroundBlur() {
-  await videoTrack.stopProcessor();
-}
-
-async function updateBlurRadius(radius) {
-  return processor.switchTo({ mode: 'background-blur', blurRadius: radius });
-}
 ```
 
-In a real application, it's likely you will want to only sometimes apply background effects. You
-could accomplish this by calling `videoTrack.setProcessor(...)` / `videoTrack.stopProcessor(...)` on
-demand, but these functions can sometimes result in output visual artifacts as part of the switching
-process, which can result in a poor user experience.
+Available modes: `background-blur`, `virtual-background`, and `disabled` (passthrough).
 
-A better option which won't result in any visual artifacts while switching is to initialize the
-`BackgroundProcessor` in its "disabled" mode, and then later on switch to the desired mode. For
-example:
-```ts
-const videoTrack = await createLocalVideoTrack();
-const processor = BackgroundProcessor({ mode: 'disabled' });
-await videoTrack.setProcessor(processor);
-room.localParticipant.publishTrack(videoTrack);
-
-async function enableBlur(radius) {
-  await processor.switchTo({ mode: 'background-blur', blurRadius: radius });
-}
-
-async function disableBlur() {
-  await processor.switchTo({ mode: 'disabled' });
-}
-```
+See [processor-docs/video-processors.md](processor-docs/video-processors.md) for full usage, browser support checks, and how to avoid visual artifacts when switching modes.
 
-## Developing your own processors
+## Audio processors
 
-A track processor is instantiated with a Transformer.
+Audio track processors work similarly — they intercept the local audio track and pipe it through a Web Audio API processing graph before publishing. The included `GainAudioProcessor` provides gain control and serves as a reference implementation for building custom audio processors:
 
 ```ts
-// src/index.ts
-export const VirtualBackground = (imagePath: string) => {
-  const pipeline = new ProcessorWrapper(new BackgroundTransformer({ imagePath }));
-  return pipeline;
-};
+import { GainAudioProcessor } from '@livekit/track-processors';
+
+const processor = new GainAudioProcessor({ gainValue: 1.5 });
+await audioTrack.setProcessor(processor);
 ```
 
-### Available base transformers
+See [processor-docs/audio-processors.md](processor-docs/audio-processors.md) for full usage, the `TrackProcessor` interface for audio, and a guide to building your own audio processor with the Web Audio API.
+
+## Developing your own processors
 
-- BackgroundTransformer (can blur background, use a virtual background, or be put into a disabled state);
+This package implements the `TrackProcessor` interface from `livekit-client`. Video and audio processors take different approaches:
 
+- **Video processors** use `ProcessorWrapper` with a transformer pipeline — see [processor-docs/video-processors.md](processor-docs/video-processors.md#developing-your-own-video-processor)
+- **Audio processors** implement `TrackProcessor` directly using the Web Audio API — see [processor-docs/audio-processors.md](processor-docs/audio-processors.md#building-your-own-audio-processor)
 
 ## Running the sample app
 
-This repository includes a small example app built on [Vite](https://vitejs.dev/). Run it with:
+This repository includes a small example app built on [Vite](https://vitejs.dev/) that demonstrates both video and audio processors. Run it with:
 
 ```
 # install pnpm: https://pnpm.io/installation

example/index.html

@@ -159,6 +159,37 @@ <h2>LiveKit track processor sample</h2>
                   </button>
                 </div>
               </div>
+
+              <div class="card mt-1 mb-1 mx-n2 p-2">
+                <div class="d-flex align-items-center justify-content-between">
+                  <button
+                    id="toggle-audio-processor"
+                    class="btn btn-secondary mr-2"
+                    disabled
+                    type="button"
+                    onclick="appActions.toggleAudioProcessorEnabled()"
+                  >
+                    Insert Audio Processor
+                  </button>
+                </div>
+
+                <div id="audio-processor-controls" style="display: none;">
+                  <div class="d-flex align-items-center mt-1">
+                    <label for="gain-slider" class="mr-2 mb-0">Gain:</label>
+                    <input
+                      type="range"
+                      id="gain-slider"
+                      min="0"
+                      max="3"
+                      value="1"
+                      step="0.1"
+                      style="flex: 1;"
+                      oninput="appActions.updateGain(parseFloat(this.value))"
+                    />
+                    <span id="gain-value" class="ml-2" style="min-width: 40px;">1.0</span>
+                  </div>
+                </div>
+              </div>
             </div>
           </div>
 

example/sample.ts

@@ -24,7 +24,7 @@ import {
   facingModeFromLocalTrack,
   setLogLevel,
 } from 'livekit-client';
-import { BackgroundProcessor, BackgroundProcessorOptions } from '../src';
+import { BackgroundProcessor, BackgroundProcessorOptions, GainAudioProcessor } from '../src';
 
 const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
 
@@ -36,6 +36,8 @@ const state = {
   bitrateInterval: undefined as any,
   isBackgroundProcessorEnabled: false,
   backgroundProcessor: BackgroundProcessor({ mode: 'background-blur', blurRadius: BLUR_RADIUS }),
+  isAudioProcessorEnabled: false,
+  gainProcessor: new GainAudioProcessor({ gainValue: 1.0 }),
 };
 let currentRoom: Room | undefined;
 
@@ -339,6 +341,44 @@ const appActions = {
     }
   },
 
+  toggleAudioProcessorEnabled: async () => {
+    if (!currentRoom) return;
+
+    setButtonDisabled('toggle-audio-processor', true);
+
+    try {
+      const micPub = currentRoom.localParticipant.getTrackPublication(Track.Source.Microphone);
+      if (!micPub || !micPub.track) {
+        appendLog('ERROR: No microphone track found. Enable audio first.');
+        return;
+      }
+      const micTrack = micPub.track as LocalAudioTrack;
+
+      if (state.isAudioProcessorEnabled) {
+        await micTrack.stopProcessor();
+        state.isAudioProcessorEnabled = false;
+      } else {
+        await micTrack.setProcessor(state.gainProcessor);
+        state.isAudioProcessorEnabled = true;
+      }
+    } catch (e: any) {
+      appendLog(`ERROR: ${e.message}`);
+    } finally {
+      updateAudioProcessorButtons();
+      if (currentRoom?.state === ConnectionState.Connected) {
+        setButtonDisabled('toggle-audio-processor', false);
+      }
+    }
+  },
+
+  updateGain: (value: number) => {
+    state.gainProcessor.setGain(value);
+    const display = $('gain-value');
+    if (display) {
+      display.textContent = value.toFixed(1);
+    }
+  },
+
   startAudio: () => {
     currentRoom?.startAudio();
   },
@@ -409,7 +449,9 @@ function handleRoomDisconnect(reason?: DisconnectReason) {
   appendLog('disconnected from room', { reason });
   setButtonsForState(false);
   state.isBackgroundProcessorEnabled = false;
+  state.isAudioProcessorEnabled = false;
   updateTrackProcessorModeButtons();
+  updateAudioProcessorButtons();
   renderParticipant(currentRoom.localParticipant, true);
   currentRoom.remoteParticipants.forEach((p) => {
     renderParticipant(p, true);
@@ -662,6 +704,7 @@ function setButtonsForState(connected: boolean) {
     'switch-to-background-blur-button',
     'switch-to-virtual-background-button',
     'switch-to-disabled-button',
+    'toggle-audio-processor',
   ];
   const disconnectedSet = ['connect-button'];
 
@@ -766,6 +809,17 @@ function updateTrackProcessorModeButtons() {
   }
 }
 
+function updateAudioProcessorButtons() {
+  const toggleButtonEnabled = currentRoom?.state === ConnectionState.Connected;
+  if (state.isAudioProcessorEnabled) {
+    setButtonState('toggle-audio-processor', 'Remove Audio Processor', false, !toggleButtonEnabled);
+    $('audio-processor-controls').style.display = 'block';
+  } else {
+    setButtonState('toggle-audio-processor', 'Insert Audio Processor', false, !toggleButtonEnabled);
+    $('audio-processor-controls').style.display = 'none';
+  }
+}
+
 async function acquireDeviceList() {
   handleDevicesChanged();
 }