From 874c72c9638824676a309287b6d157dec5cfa667 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 09:57:11 +0200 Subject: [PATCH 01/12] Expose frame processing stats via optional callback --- example/sample.ts | 14 ++++++- package.json | 14 +++---- pnpm-lock.yaml | 45 +++++++++++++++-------- src/index.ts | 20 ++++++++-- src/transformers/BackgroundTransformer.ts | 18 +++++++++ 5 files changed, 82 insertions(+), 29 deletions(-) diff --git a/example/sample.ts b/example/sample.ts index 7bb2b86..4dc216b 100644 --- a/example/sample.ts +++ b/example/sample.ts @@ -31,8 +31,18 @@ const $ = (id: string) => document.getElementById(id) as const state = { defaultDevices: new Map(), bitrateInterval: undefined as any, - blur: BackgroundBlur(10, { delegate: 'GPU' }), - virtualBackground: VirtualBackground('/samantha-gades-BlIhVfXbi9s-unsplash.jpg'), + blur: BackgroundBlur(10, { delegate: 'GPU' }, (stats) => { + console.log('frame processing stats', stats); + }), + virtualBackground: VirtualBackground( + '/samantha-gades-BlIhVfXbi9s-unsplash.jpg', + { + delegate: 'GPU', + }, + (stats) => { + console.log('frame processing stats', stats); + }, + ), }; let currentRoom: Room | undefined; diff --git a/package.json b/package.json index 5ee2ba7..3ee5e92 100644 --- a/package.json +++ b/package.json @@ -30,19 +30,19 @@ "devDependencies": { "@changesets/cli": "^2.26.2", "@livekit/changesets-changelog-github": "^0.0.4", - "@trivago/prettier-plugin-sort-imports": "^4.1.1", - "@types/dom-mediacapture-transform": "^0.1.6", - "@types/offscreencanvas": "^2019.7.0", + "@trivago/prettier-plugin-sort-imports": "^4.2.1", + "@types/dom-mediacapture-transform": "^0.1.9", + "@types/offscreencanvas": "^2019.7.3", "@typescript-eslint/eslint-plugin": "^5.62.0", "eslint": "8.39.0", "eslint-config-airbnb-typescript": "17.0.0", "eslint-config-prettier": "8.8.0", - "eslint-plugin-ecmascript-compat": "^3.0.0", + "eslint-plugin-ecmascript-compat": "^3.1.0", "eslint-plugin-import": "2.27.5", "prettier": "^2.8.8", - "tsup": "^7.1.0", - "typescript": "^5.0.4", - "vite": "^4.3.8" + "tsup": "^7.2.0", + "typescript": "^5.2.2", + "vite": "^4.5.0" }, "packageManager": "pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bac980f..bedf1c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 0.10.21 livekit-client: specifier: ^1.12.0 || ^2.1.0 - version: 2.9.9 + version: 2.11.2 devDependencies: '@changesets/cli': specifier: ^2.26.2 @@ -22,13 +22,13 @@ importers: specifier: ^0.0.4 version: 0.0.4 '@trivago/prettier-plugin-sort-imports': - specifier: ^4.1.1 + specifier: ^4.2.1 version: 4.2.1(prettier@2.8.8) '@types/dom-mediacapture-transform': - specifier: ^0.1.6 + specifier: ^0.1.9 version: 0.1.9 '@types/offscreencanvas': - specifier: ^2019.7.0 + specifier: ^2019.7.3 version: 2019.7.3 '@typescript-eslint/eslint-plugin': specifier: ^5.62.0 @@ -43,7 +43,7 @@ importers: specifier: 8.8.0 version: 8.8.0(eslint@8.39.0) eslint-plugin-ecmascript-compat: - specifier: ^3.0.0 + specifier: ^3.1.0 version: 3.1.0(eslint@8.39.0) eslint-plugin-import: specifier: 2.27.5 @@ -52,13 +52,13 @@ importers: specifier: ^2.8.8 version: 2.8.8 tsup: - specifier: ^7.1.0 + specifier: ^7.2.0 version: 7.2.0(postcss@8.4.31)(typescript@5.2.2) typescript: - specifier: ^5.0.4 + specifier: ^5.2.2 version: 5.2.2 vite: - specifier: ^4.3.8 + specifier: ^4.5.0 version: 4.5.0 packages: @@ -375,8 +375,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.34.0': - resolution: {integrity: sha512-bU7pCLAMRVTVZb1KSxA46q55bhOc4iATrY/gccy2/oX1D57tiZEI+8wGRWHeDwBb0UwnABu6JXzC4tTFkdsaOg==} + '@livekit/protocol@1.36.1': + resolution: {integrity: sha512-nN3QnITAQ5yXk7UKfotH7CRWIlEozNWeKVyFJ0/+dtSzvWP/ib+10l1DDnRYi3A1yICJOGAKFgJ5d6kmi1HCUA==} '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -719,6 +719,15 @@ packages: supports-color: optional: true + debug@4.4.0: + resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -1311,8 +1320,8 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - livekit-client@2.9.9: - resolution: {integrity: sha512-Mrm9Z/kPmJP5r4EMbzXPfB27gRP+tZtU7Zzen2o8u6w5N6FxaqXerRFqtXddGXaVzZouJOAg51/5A9u3saC/2A==} + livekit-client@2.11.2: + resolution: {integrity: sha512-VndcZvUC37/tTHT8sK15pUMk7TdRfQ2mzXeVxIwkpLZRfFF9EM57h084+JEpiQg7EUzRWevN8YEBMvHFGx25TA==} load-tsconfig@0.2.5: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} @@ -2455,7 +2464,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.34.0': + '@livekit/protocol@1.36.1': dependencies: '@bufbuild/protobuf': 1.10.0 @@ -2551,7 +2560,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.2.2) - debug: 4.3.4 + debug: 4.4.0 eslint: 8.39.0 optionalDependencies: typescript: 5.2.2 @@ -2839,6 +2848,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.4.0: + dependencies: + ms: 2.1.3 + decamelize-keys@1.1.1: dependencies: decamelize: 1.2.0 @@ -3517,10 +3530,10 @@ snapshots: lines-and-columns@1.2.4: {} - livekit-client@2.9.9: + livekit-client@2.11.2: dependencies: '@livekit/mutex': 1.1.1 - '@livekit/protocol': 1.34.0 + '@livekit/protocol': 1.36.1 events: 3.3.0 loglevel: 1.9.2 sdp-transform: 2.15.0 diff --git a/src/index.ts b/src/index.ts index b67b869..78a260c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ import ProcessorWrapper from './ProcessorWrapper'; import BackgroundTransformer, { BackgroundOptions, + FrameProcessingStats, SegmenterOptions, } from './transformers/BackgroundTransformer'; @@ -8,12 +9,23 @@ export * from './transformers/types'; export { default as VideoTransformer } from './transformers/VideoTransformer'; export { ProcessorWrapper, type BackgroundOptions, type SegmenterOptions, BackgroundTransformer }; -export const BackgroundBlur = (blurRadius: number = 10, segmenterOptions?: SegmenterOptions) => { - return BackgroundProcessor({ blurRadius, segmenterOptions }, 'background-blur'); +export const BackgroundBlur = ( + blurRadius: number = 10, + segmenterOptions?: SegmenterOptions, + onFrameProcessed?: (stats: FrameProcessingStats) => void, +) => { + return BackgroundProcessor({ blurRadius, segmenterOptions, onFrameProcessed }, 'background-blur'); }; -export const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterOptions) => { - return BackgroundProcessor({ imagePath, segmenterOptions }, 'virtual-background'); +export const VirtualBackground = ( + imagePath: string, + segmenterOptions?: SegmenterOptions, + onFrameProcessed?: (stats: FrameProcessingStats) => void, +) => { + return BackgroundProcessor( + { imagePath, segmenterOptions, onFrameProcessed }, + 'virtual-background', + ); }; export const BackgroundProcessor = (options: BackgroundOptions, name = 'background-processor') => { diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index c927740..bddd700 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -5,6 +5,12 @@ import { VideoTransformerInitOptions } from './types'; export type SegmenterOptions = Partial; +export interface FrameProcessingStats { + processingTimeMs: number; + segmentationTimeMs: number; + filterTimeMs: number; +} + export type BackgroundOptions = { blurRadius?: number; imagePath?: string; @@ -12,6 +18,8 @@ export type BackgroundOptions = { segmenterOptions?: SegmenterOptions; /** cannot be updated through the `update` method, needs a restart */ assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string }; + /** called when a new frame is processed */ + onFrameProcessed?: (stats: FrameProcessingStats) => void; }; export default class BackgroundProcessor extends VideoTransformer { @@ -97,11 +105,13 @@ export default class BackgroundProcessor extends VideoTransformer (this.segmentationResults = result), ); + const segmentationTimeMs = performance.now() - startTimeMs; if (this.blurRadius) { await this.blurBackground(frame); @@ -111,7 +121,15 @@ export default class BackgroundProcessor extends VideoTransformer Date: Mon, 14 Apr 2025 12:57:25 +0200 Subject: [PATCH 02/12] wip somewhat working --- src/ProcessorWrapper.ts | 4 +- src/transformers/BackgroundTransformer.ts | 173 +++++------ src/transformers/VideoTransformer.ts | 19 +- src/utils.ts | 338 ++++++++++++++++++++++ 4 files changed, 445 insertions(+), 89 deletions(-) diff --git a/src/ProcessorWrapper.ts b/src/ProcessorWrapper.ts index da733e5..1cae8db 100644 --- a/src/ProcessorWrapper.ts +++ b/src/ProcessorWrapper.ts @@ -78,6 +78,7 @@ export default class ProcessorWrapper console.error('error when trying to pipe', e)) .finally(() => this.destroy()); + this.processedTrack = this.trackGenerator as MediaStreamVideoTrack; } @@ -96,7 +97,8 @@ export default class ProcessorWrapper 0) { - this.ctx.globalCompositeOperation = 'copy'; - - this.ctx.putImageData( - maskToImageData( - this.segmentationResults.categoryMask, - this.segmentationResults.categoryMask.width, - this.segmentationResults.categoryMask.height, - ), - 0, - 0, - ); - this.ctx.filter = 'none'; - this.ctx.globalCompositeOperation = 'source-in'; - if (this.backgroundImage) { - this.ctx.drawImage( - this.backgroundImage, - 0, - 0, - this.backgroundImage.width, - this.backgroundImage.height, - 0, - 0, - this.canvas.width, - this.canvas.height, - ); - } else { - this.ctx.fillStyle = '#00FF00'; - this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height); - } - - this.ctx.globalCompositeOperation = 'destination-over'; - } - this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); + async drawFrame(frame: VideoFrame) { + if (!this.canvas || !this.gl || !this.segmentationResults || !this.inputVideo) return; + this.gl.render(frame, 10, this.segmentationResults.categoryMask?.getAsWebGLTexture()); } - async blurBackground(frame: VideoFrame) { - if ( - !this.ctx || - !this.canvas || - !this.segmentationResults?.categoryMask?.canvas || - !this.inputVideo - ) { - return; - } - - this.ctx.save(); - this.ctx.globalCompositeOperation = 'copy'; - - if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) { - this.ctx.putImageData( - maskToImageData( - this.segmentationResults.categoryMask, - this.segmentationResults.categoryMask.width, - this.segmentationResults.categoryMask.height, - ), - 0, - 0, - ); - this.ctx.filter = 'none'; - this.ctx.globalCompositeOperation = 'source-out'; - this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - this.ctx.globalCompositeOperation = 'destination-over'; - this.ctx.filter = `blur(${this.blurRadius}px)`; - this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - this.ctx.restore(); - } - } + // async drawVirtualBackground(frame: VideoFrame) { + // if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return; + // // this.ctx.save(); + // // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + // if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) { + // this.ctx.globalCompositeOperation = 'copy'; + + // this.ctx.putImageData( + // maskToImageData( + // this.segmentationResults.categoryMask, + // this.segmentationResults.categoryMask.width, + // this.segmentationResults.categoryMask.height, + // ), + // 0, + // 0, + // ); + // this.ctx.filter = 'none'; + // this.ctx.globalCompositeOperation = 'source-in'; + // if (this.backgroundImage) { + // this.ctx.drawImage( + // this.backgroundImage, + // 0, + // 0, + // this.backgroundImage.width, + // this.backgroundImage.height, + // 0, + // 0, + // this.canvas.width, + // this.canvas.height, + // ); + // } else { + // this.ctx.fillStyle = '#00FF00'; + // this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height); + // } + + // this.ctx.globalCompositeOperation = 'destination-over'; + // } + // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); + // } + + // async blurBackground(frame: VideoFrame) { + // if ( + // !this.ctx || + // !this.canvas || + // !this.segmentationResults?.categoryMask?.canvas || + // !this.inputVideo + // ) { + // return; + // } + + // this.ctx.save(); + // this.ctx.globalCompositeOperation = 'copy'; + + // if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) { + // this.ctx.putImageData( + // maskToImageData( + // this.segmentationResults.categoryMask, + // this.segmentationResults.categoryMask.width, + // this.segmentationResults.categoryMask.height, + // ), + // 0, + // 0, + // ); + // this.ctx.filter = 'none'; + // this.ctx.globalCompositeOperation = 'source-out'; + // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); + // this.ctx.globalCompositeOperation = 'destination-over'; + // this.ctx.filter = `blur(${this.blurRadius}px)`; + // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); + // this.ctx.restore(); + // } + // } } -function maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData { - const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4); - const result = mask.getAsUint8Array(); - for (let i = 0; i < result.length; i += 1) { - const offset = i * 4; - dataArray[offset] = result[i]; - dataArray[offset + 1] = result[i]; - dataArray[offset + 2] = result[i]; - dataArray[offset + 3] = result[i]; - } - return new ImageData(dataArray, videoWidth, videoHeight); -} +// function maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData { +// const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4); +// const result = mask.getAsUint8Array(); +// for (let i = 0; i < result.length; i += 1) { +// const offset = i * 4; +// dataArray[offset] = result[i]; +// dataArray[offset + 1] = result[i]; +// dataArray[offset + 2] = result[i]; +// dataArray[offset + 3] = result[i]; +// } +// return new ImageData(dataArray, videoWidth, videoHeight); +// } diff --git a/src/transformers/VideoTransformer.ts b/src/transformers/VideoTransformer.ts index 996278b..c3857b9 100644 --- a/src/transformers/VideoTransformer.ts +++ b/src/transformers/VideoTransformer.ts @@ -1,3 +1,4 @@ +import { createBlurPipeline } from '../utils'; import { VideoTrackTransformer, VideoTransformerInitOptions } from './types'; export default abstract class VideoTransformer> @@ -7,10 +8,12 @@ export default abstract class VideoTransformer; + protected isDisabled?: Boolean = false; async init({ @@ -26,7 +29,10 @@ export default abstract class VideoTransformer void; + cleanup: () => void; +}; + +export function initWebGL(canvas: OffscreenCanvas): WebGLRenderer { + const gl = canvas.getContext('webgl'); + if (!gl) throw new Error('WebGL not supported'); + + const vsSource = ` + attribute vec2 a_position; + attribute vec2 a_texCoord; + varying vec2 v_texCoord; + void main() { + gl_Position = vec4(a_position, 0, 1); + v_texCoord = a_texCoord; + } + `; + + const fsSource = ` + precision mediump float; + varying vec2 v_texCoord; + uniform sampler2D u_texture; + void main() { + gl_FragColor = texture2D(u_texture, v_texCoord); + } + `; + + const createShader = (type: number, source: string): WebGLShader => { + const shader = gl.createShader(type); + if (!shader) throw new Error('Failed to create shader'); + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error(gl.getShaderInfoLog(shader) || 'Unknown shader error'); + } + return shader; + }; + + const vertexShader = createShader(gl.VERTEX_SHADER, vsSource); + const fragmentShader = createShader(gl.FRAGMENT_SHADER, fsSource); + + const program = gl.createProgram(); + if (!program) throw new Error('Failed to create program'); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error(gl.getProgramInfoLog(program) || 'Program linking failed'); + } + gl.useProgram(program); + + // Position buffer + const positionBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + const positions = new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]); + gl.bufferData(gl.ARRAY_BUFFER, positions, gl.STATIC_DRAW); + + const posLoc = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(posLoc); + gl.vertexAttribPointer(posLoc, 2, gl.FLOAT, false, 0, 0); + + // Texture coord buffer + const texCoordBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + const texCoords = new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]); + gl.bufferData(gl.ARRAY_BUFFER, texCoords, gl.STATIC_DRAW); + + const texLoc = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texLoc); + gl.vertexAttribPointer(texLoc, 2, gl.FLOAT, false, 0, 0); + + // Texture setup + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + + const render = (videoFrame: VideoFrame): void => { + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.viewport(0, 0, canvas.width, canvas.height); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.drawArrays(gl.TRIANGLES, 0, 6); + }; + + const cleanup = (): void => { + gl.deleteTexture(texture); + gl.deleteBuffer(positionBuffer); + gl.deleteBuffer(texCoordBuffer); + gl.deleteProgram(program); + gl.deleteShader(vertexShader); + gl.deleteShader(fragmentShader); + }; + + return { gl, render, cleanup }; +} + +type BlurPipeline = { + render: (frame: VideoFrame, radius: number, mask?: TexImageSource) => void; + cleanup: () => void; +}; + +export function createBlurPipeline(canvas: OffscreenCanvas): BlurPipeline { + const gl = canvas.getContext('webgl')!; + if (!gl) throw new Error('WebGL not supported'); + + const baseVertexShader = ` + attribute vec2 a_position; + attribute vec2 a_texCoord; + varying vec2 v_texCoord; + uniform float u_flipY; + + void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + v_texCoord = vec2(a_texCoord.x, u_flipY == 1.0 ? 1.0 - a_texCoord.y : a_texCoord.y); + } + `; + + const blurFragmentShader = ` + precision mediump float; + varying vec2 v_texCoord; + uniform sampler2D u_texture; + uniform vec2 u_texelSize; + uniform vec2 u_direction; + uniform float u_radius; + + void main() { + float sigma = u_radius; + float twoSigmaSq = 2.0 * sigma * sigma; + float totalWeight = 0.0; + vec3 result = vec3(0.0); + const int MAX_SAMPLES = 16; + int radius = int(min(float(MAX_SAMPLES), ceil(u_radius))); + + for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) { + float offset = float(i); + if (abs(offset) > float(radius)) continue; + float weight = exp(-(offset * offset) / twoSigmaSq); + vec2 sampleCoord = v_texCoord + u_direction * u_texelSize * offset; + result += texture2D(u_texture, sampleCoord).rgb * weight; + totalWeight += weight; + } + + gl_FragColor = vec4(result / totalWeight, 1.0); + } + `; + + const compositeFragmentShader = ` + precision mediump float; + varying vec2 v_texCoord; + uniform sampler2D u_original; + uniform sampler2D u_blurred; + uniform sampler2D u_mask; + + void main() { + vec4 orig = texture2D(u_original, v_texCoord); + vec4 blur = texture2D(u_blurred, v_texCoord); + float mask = texture2D(u_mask, v_texCoord).r; + gl_FragColor = mix(blur, orig, mask); + } +`; + + const compositeProgram = createProgram(baseVertexShader, compositeFragmentShader); + + // --- Compile + function compileShader(type: number, source: string): WebGLShader { + const shader = gl.createShader(type)!; + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error(gl.getShaderInfoLog(shader)!); + } + return shader; + } + + function createProgram(vsSrc: string, fsSrc: string): WebGLProgram { + const vs = compileShader(gl.VERTEX_SHADER, vsSrc); + const fs = compileShader(gl.FRAGMENT_SHADER, fsSrc); + const program = gl.createProgram()!; + gl.attachShader(program, vs); + gl.attachShader(program, fs); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error(gl.getProgramInfoLog(program)!); + } + return program; + } + + const program = createProgram(baseVertexShader, blurFragmentShader); + + const quad = new Float32Array([ + -1, -1, 0, 1, 1, -1, 1, 1, -1, 1, 0, 0, -1, 1, 0, 0, 1, -1, 1, 1, 1, 1, 1, 0, + ]); + + const quadBuffer = gl.createBuffer()!; + gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); + gl.bufferData(gl.ARRAY_BUFFER, quad, gl.STATIC_DRAW); + + const positionLoc = gl.getAttribLocation(program, 'a_position'); + const texCoordLoc = gl.getAttribLocation(program, 'a_texCoord'); + + const u_texture = gl.getUniformLocation(program, 'u_texture'); + const u_texelSize = gl.getUniformLocation(program, 'u_texelSize'); + const u_direction = gl.getUniformLocation(program, 'u_direction'); + const u_radius = gl.getUniformLocation(program, 'u_radius'); + const u_flipY = gl.getUniformLocation(program, 'u_flipY'); + + const texture = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + + const framebuffer = gl.createFramebuffer()!; + gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + // Second texture and framebuffer for vertical blur pass + const blurTexture = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, blurTexture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + canvas.width, + canvas.height, + 0, + gl.RGBA, + gl.UNSIGNED_BYTE, + null, + ); + + const blurFBO = gl.createFramebuffer()!; + gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, blurTexture, 0); + + const maskTex = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, maskTex); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + + // === Render Function === + function render(frame: VideoFrame, radius: number, mask?: TexImageSource) { + canvas.width = frame.displayWidth; + canvas.height = frame.displayHeight; + + const texW = canvas.width; + const texH = canvas.height; + + gl.viewport(0, 0, texW, texH); + + // Upload frame to texture + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); + + gl.useProgram(program); + gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); + gl.enableVertexAttribArray(positionLoc); + gl.enableVertexAttribArray(texCoordLoc); + gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 16, 0); + gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 16, 8); + + // === Pass 1: Horizontal Blur === + gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.uniform1i(u_texture, 0); + gl.uniform2f(u_texelSize, 1 / texW, 1 / texH); + gl.uniform2f(u_direction, 1.0, 0.0); // horizontal + gl.uniform1f(u_radius, radius); + gl.uniform1f(u_flipY, 0.0); + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // === Pass 2: Vertical Blur === + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.bindTexture(gl.TEXTURE_2D, blurTexture); + gl.uniform2f(u_direction, 0.0, 1.0); // vertical + gl.uniform1f(u_flipY, 1.0); + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // === Composite === + gl.bindTexture(gl.TEXTURE_2D, maskTex); + if (mask) { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, gl.LUMINANCE, gl.UNSIGNED_BYTE, mask); + } + // === Final Pass: Composite original + blurred using mask === + gl.useProgram(compositeProgram); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.viewport(0, 0, canvas.width, canvas.height); + gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); + gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 16, 0); + gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 16, 8); + + // Bind original texture + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_original'), 0); + + // Bind blurred texture + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, blurTexture); + gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_blurred'), 1); + + // Bind mask texture + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, maskTex); + gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_mask'), 2); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + } + + function cleanup() { + gl.deleteBuffer(quadBuffer); + gl.deleteTexture(texture); + gl.deleteFramebuffer(framebuffer); + gl.deleteTexture(blurTexture); + gl.deleteFramebuffer(blurFBO); + } + + return { render, cleanup }; +} From b16c53d7751acc6679714e3285a87beac5ec0b7e Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 16:48:33 +0200 Subject: [PATCH 03/12] wip --- src/transformers/BackgroundTransformer.ts | 70 +++-- src/transformers/VideoTransformer.ts | 13 +- src/utils.ts | 135 ++++---- src/webgl.ts | 219 +++++++++++++ src/webgl/index.ts | 363 ++++++++++++++++++++++ 5 files changed, 714 insertions(+), 86 deletions(-) create mode 100644 src/webgl.ts create mode 100644 src/webgl/index.ts diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index 431f777..2f350ac 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -57,8 +57,8 @@ export default class BackgroundProcessor extends VideoTransformer) { try { - if (!(frame instanceof VideoFrame)) { + if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) { console.debug('empty frame detected, ignoring'); return; } + if (this.isDisabled) { controller.enqueue(frame); return; @@ -104,34 +105,45 @@ export default class BackgroundProcessor extends VideoTransformer (this.segmentationResults = result), - ); - const segmentationTimeMs = performance.now() - startTimeMs; - - if (this.blurRadius) { - await this.drawFrame(frame); - } else { - await this.drawFrame(frame); - } - const newFrame = new VideoFrame(this.canvas, { - timestamp: frame.timestamp || Date.now(), + this.imageSegmenter?.segmentForVideo(frame, startTimeMs, (result) => { + const segmentationTimeMs = performance.now() - startTimeMs; + this.segmentationResults = result; + this.drawFrame(frame); + if (this.canvas && this.canvas.width > 0 && this.canvas.height > 0) { + const newFrame = new VideoFrame(this.canvas, { + timestamp: frame.timestamp || Date.now(), + }); + const filterTimeMs = performance.now() - startTimeMs - segmentationTimeMs; + const stats: FrameProcessingStats = { + processingTimeMs: performance.now() - startTimeMs, + segmentationTimeMs, + filterTimeMs, + }; + this.options.onFrameProcessed?.(stats); + + controller.enqueue(newFrame); + } else { + controller.enqueue(frame); + } + frame.close(); }); - const filterTimeMs = performance.now() - startTimeMs - segmentationTimeMs; - - controller.enqueue(newFrame); - const stats: FrameProcessingStats = { - processingTimeMs: performance.now() - startTimeMs, - segmentationTimeMs, - filterTimeMs, - }; - this.options.onFrameProcessed?.(stats); + + // if (this.blurRadius) { + // await this.drawFrame(frame); + // } else { + // await this.drawFrame(frame); + // } + // const newFrame = new VideoFrame(this.canvas, { + // timestamp: frame.timestamp || Date.now(), + // }); + + // controller.enqueue(newFrame); } finally { - frame?.close(); + // frame?.close(); } } @@ -146,7 +158,11 @@ export default class BackgroundProcessor extends VideoTransformer> @@ -12,7 +12,7 @@ export default abstract class VideoTransformer; + gl?: ReturnType; protected isDisabled?: Boolean = false; @@ -30,7 +30,7 @@ export default abstract class VideoTransformer void; + render: (frame: VideoFrame, radius: number, mask: MPMask) => void; cleanup: () => void; }; @@ -189,8 +191,8 @@ export function createBlurPipeline(canvas: OffscreenCanvas): BlurPipeline { void main() { vec4 orig = texture2D(u_original, v_texCoord); vec4 blur = texture2D(u_blurred, v_texCoord); - float mask = texture2D(u_mask, v_texCoord).r; - gl_FragColor = mix(blur, orig, mask); + vec4 mask = texture2D(u_mask, v_texCoord); + gl_FragColor = vec4(mask.r * 10.0, mask.g * 10.0, mask.b * 10.0, orig.a); } `; @@ -270,83 +272,112 @@ export function createBlurPipeline(canvas: OffscreenCanvas): BlurPipeline { null, ); + // const maskTexture = gl.createTexture()!; + // gl.bindTexture(gl.TEXTURE_2D, maskTexture); + // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + const blurFBO = gl.createFramebuffer()!; gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, blurTexture, 0); - const maskTex = gl.createTexture()!; - gl.bindTexture(gl.TEXTURE_2D, maskTex); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - // === Render Function === - function render(frame: VideoFrame, radius: number, mask?: TexImageSource) { - canvas.width = frame.displayWidth; - canvas.height = frame.displayHeight; - - const texW = canvas.width; - const texH = canvas.height; + function render(frame: VideoFrame, radius: number, mask: MPMask) { + if (!mask.canvas) { + console.warn('MPMask does not have a canvas, skipping render'); + return; + } - gl.viewport(0, 0, texW, texH); + const maskContext = mask.canvas.getContext('webgl') as WebGLRenderingContext | null; + if (!maskContext) { + console.warn('MPMask canvas does not have WebGL context, skipping render'); + return; + } - // Upload frame to texture - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); + // Bind the framebuffer for horizontal blur + gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer); + gl.viewport(0, 0, canvas.width, canvas.height); + // Use the blur program for horizontal pass gl.useProgram(program); + + // Set up vertex attributes gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); gl.enableVertexAttribArray(positionLoc); - gl.enableVertexAttribArray(texCoordLoc); gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 16, 0); + gl.enableVertexAttribArray(texCoordLoc); gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 16, 8); - // === Pass 1: Horizontal Blur === - gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); - gl.bindTexture(gl.TEXTURE_2D, texture); + // Set uniforms for horizontal blur gl.uniform1i(u_texture, 0); - gl.uniform2f(u_texelSize, 1 / texW, 1 / texH); - gl.uniform2f(u_direction, 1.0, 0.0); // horizontal + gl.uniform2f(u_texelSize, 1.0 / canvas.width, 1.0 / canvas.height); + gl.uniform2f(u_direction, 1.0, 0.0); gl.uniform1f(u_radius, radius); - gl.uniform1f(u_flipY, 0.0); - gl.drawArrays(gl.TRIANGLES, 0, 6); - - // === Pass 2: Vertical Blur === - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.bindTexture(gl.TEXTURE_2D, blurTexture); - gl.uniform2f(u_direction, 0.0, 1.0); // vertical gl.uniform1f(u_flipY, 1.0); + + // Bind and upload the frame to texture + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); + + // Draw horizontal blur gl.drawArrays(gl.TRIANGLES, 0, 6); - // === Composite === - gl.bindTexture(gl.TEXTURE_2D, maskTex); - if (mask) { - gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, gl.LUMINANCE, gl.UNSIGNED_BYTE, mask); - } - // === Final Pass: Composite original + blurred using mask === - gl.useProgram(compositeProgram); - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.viewport(0, 0, canvas.width, canvas.height); - gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); - gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 16, 0); - gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 16, 8); + // Bind the framebuffer for vertical blur + gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); + + // Set uniforms for vertical blur + gl.uniform2f(u_direction, 0.0, 1.0); - // Bind original texture + // Bind the horizontally blurred texture gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); + + // Draw vertical blur + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // Now use the composite program to blend with mask + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.useProgram(compositeProgram); + + // Set up vertex attributes for composite + gl.enableVertexAttribArray(gl.getAttribLocation(compositeProgram, 'a_position')); + gl.vertexAttribPointer( + gl.getAttribLocation(compositeProgram, 'a_position'), + 2, + gl.FLOAT, + false, + 16, + 0, + ); + gl.enableVertexAttribArray(gl.getAttribLocation(compositeProgram, 'a_texCoord')); + gl.vertexAttribPointer( + gl.getAttribLocation(compositeProgram, 'a_texCoord'), + 2, + gl.FLOAT, + false, + 16, + 8, + ); + + // Set uniforms for composite gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_original'), 0); + gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_blurred'), 1); + gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_mask'), 2); - // Bind blurred texture + // Bind textures + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, blurTexture); - gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_blurred'), 1); - - // Bind mask texture gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, maskTex); - gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_mask'), 2); + // The mask texture is already bound to mask.canvas, so we can use it directly + const maskTexture = maskContext.getParameter(maskContext.TEXTURE_BINDING_2D); + gl.bindTexture(gl.TEXTURE_2D, maskTexture); + // Draw final composite gl.drawArrays(gl.TRIANGLES, 0, 6); } diff --git a/src/webgl.ts b/src/webgl.ts new file mode 100644 index 0000000..3ae32d5 --- /dev/null +++ b/src/webgl.ts @@ -0,0 +1,219 @@ +import { FilesetResolver, ImageSegmenter, ImageSegmenterResult } from '@mediapipe/tasks-vision'; + +let videoTexture: WebGLTexture, bgTexture: WebGLTexture; + +let bgImage = 'https://videos.electroteque.org/textures/virtualbg.jpg'; + +function initBkgnd( + gl: WebGL2RenderingContext, + bgImage: string, + bgTextureLocation: WebGLUniformLocation, +) { + bgTexture = initTexture(gl, 0); + gl.uniform1i(bgTextureLocation, 0); + + if (bgImage) { + const img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = () => { + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + //resizeForTexture(img, img.width, img.height); + console.log('FILL BACKGROUND'); + fillBackgroundImage(gl, img); + }; + img.src = bgImage; + } else { + // Fill with black background + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clear(gl.COLOR_BUFFER_BIT); + } +} + +function fillBackgroundImage(gl: WebGL2RenderingContext, img: HTMLImageElement) { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img); + // gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); +} + +const createShaderProgram = (gl: WebGL2RenderingContext) => { + const vs = ` + attribute vec2 position; + varying vec2 texCoords; + + void main() { + texCoords = (position + 1.0) / 2.0; + texCoords.y = 1.0 - texCoords.y; + gl_Position = vec4(position, 0, 1.0); + } + `; + + const fs = ` + precision highp float; + varying vec2 texCoords; + uniform sampler2D background; + uniform sampler2D frame; + uniform sampler2D mask; + void main() { + vec4 maskTex = texture2D(mask, texCoords); + vec4 frameTex = texture2D(frame, texCoords); + vec4 bgTex = texture2D(background, texCoords); + + + float a = maskTex.r; + + gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), a); + //gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - a); + + } + `; + + const vertexShader = gl.createShader(gl.VERTEX_SHADER); + if (!vertexShader) { + throw Error('can not create vertex shader'); + } + gl.shaderSource(vertexShader, vs); + gl.compileShader(vertexShader); + + // Create our fragment shader + const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER); + if (!fragmentShader) { + throw Error('can not create fragment shader'); + } + gl.shaderSource(fragmentShader, fs); + gl.compileShader(fragmentShader); + + // Create our program + const program = gl.createProgram(); + if (!program) { + throw Error('can not create program'); + } + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + + //bgTexture = initTexture(gl, 0); + initBkgnd(gl, bgImage, gl.getUniformLocation(program, 'background')!); + videoTexture = initTexture(gl, 1); + + return { + vertexShader, + fragmentShader, + shaderProgram: program, + attribLocations: { + position: gl.getAttribLocation(program, 'position'), + }, + uniformLocations: { + mask: gl.getUniformLocation(program, 'mask')!, + frame: gl.getUniformLocation(program, 'frame')!, + background: gl.getUniformLocation(program, 'background')!, + }, + }; +}; + +const createVertexBuffer = (gl: WebGL2RenderingContext) => { + if (!gl) { + return null; + } + const vertexBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]), + gl.STATIC_DRAW, + ); + return vertexBuffer; +}; + +function initTexture(gl: WebGL2RenderingContext, texIndex: number) { + const texRef = gl.TEXTURE0 + texIndex; + gl.activeTexture(gl.TEXTURE0 + texIndex); + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + //gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + + gl.bindTexture(gl.TEXTURE_2D, texture); + + return texture; +} + +function createCopyTextureToCanvas(canvas: HTMLCanvasElement) { + const gl = canvas.getContext('webgl2', { premultipliedAlpha: false }) as WebGL2RenderingContext; + + gl.enable(gl.BLEND); + gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); + + if (!gl) { + return undefined; + } + const { + shaderProgram, + attribLocations: { position: positionLocation }, + uniformLocations: { + mask: maskTextureLocation, + frame: frameTextureLocation, + background: bgTextureLocation, + }, + } = createShaderProgram(gl); + const vertexBuffer = createVertexBuffer(gl); + + gl.uniform1i(bgTextureLocation, 0); + gl.uniform1i(frameTextureLocation, 1); + gl.uniform1i(maskTextureLocation, 2); + + return (mask: { getAsWebGLTexture: () => WebGLTexture }) => { + //gl.viewport(0, 0, canvas.width, canvas.height) + gl.clearColor(1.0, 1.0, 1.0, 1.0); + gl.useProgram(shaderProgram); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.viewport(0, 0, video.videoWidth, video.videoHeight); + //gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); + + const texture = mask.getAsWebGLTexture(); + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); + gl.enableVertexAttribArray(positionLocation); + + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + //gl.uniform1i(bgTextureLocation, 0) + + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.uniform1i(maskTextureLocation, 2); + + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, videoTexture); + gl.uniform1i(frameTextureLocation, 1); + + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + + return createImageBitmap(canvas); + }; +} + +const tasksCanvas = new OffscreenCanvas(1, 1); +const createImageSegmenter = async () => { + const audio = await FilesetResolver.forVisionTasks( + 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm', + ); + + imageSegmenter = await ImageSegmenter.createFromOptions(audio, { + baseOptions: { + modelAssetPath: + //"https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/latest/deeplab_v3.tflite", + 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', + delegate: 'GPU', + }, + canvas: tasksCanvas, + runningMode: 'VIDEO', + outputConfidenceMasks: true, + }); +}; +createImageSegmenter(); diff --git a/src/webgl/index.ts b/src/webgl/index.ts new file mode 100644 index 0000000..1fa4dea --- /dev/null +++ b/src/webgl/index.ts @@ -0,0 +1,363 @@ +import { MPMask } from '@mediapipe/tasks-vision'; + +// Define the blur fragment shader +const blurFragmentShader = ` + precision highp float; + varying vec2 texCoords; + uniform sampler2D u_texture; + uniform vec2 u_texelSize; + uniform vec2 u_direction; + uniform float u_radius; + + void main() { + float sigma = u_radius; + float twoSigmaSq = 2.0 * sigma * sigma; + float totalWeight = 0.0; + vec3 result = vec3(0.0); + const int MAX_SAMPLES = 16; + int radius = int(min(float(MAX_SAMPLES), ceil(u_radius))); + + for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) { + float offset = float(i); + if (abs(offset) > float(radius)) continue; + float weight = exp(-(offset * offset) / twoSigmaSq); + vec2 sampleCoord = texCoords + u_direction * u_texelSize * offset; + result += texture2D(u_texture, sampleCoord).rgb * weight; + totalWeight += weight; + } + + gl_FragColor = vec4(result / totalWeight, 1.0); + } +`; + +interface ShaderProgramOptions { + enableBlur?: boolean; +} + +const createShaderProgram = (gl: WebGL2RenderingContext, options: ShaderProgramOptions = {}) => { + const vs = ` + attribute vec2 position; + varying vec2 texCoords; + + void main() { + texCoords = (position + 1.0) / 2.0; + texCoords.y = 1.0 - texCoords.y; + gl_Position = vec4(position, 0, 1.0); + } + `; + + const cS = ` + precision highp float; + varying vec2 texCoords; + uniform sampler2D background; + uniform sampler2D frame; + uniform sampler2D mask; + void main() { + vec4 maskTex = texture2D(mask, texCoords); + vec4 frameTex = texture2D(frame, texCoords); + vec4 bgTex = texture2D(background, texCoords); + + + float a = maskTex.r; + + //gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), a); + gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - a); + + } + `; + + const vertexShader = gl.createShader(gl.VERTEX_SHADER); + if (!vertexShader) { + throw Error('can not create vertex shader'); + } + gl.shaderSource(vertexShader, vs); + gl.compileShader(vertexShader); + + // Create our fragment shader + const compositeShader = gl.createShader(gl.FRAGMENT_SHADER); + if (!compositeShader) { + throw Error('can not create fragment shader'); + } + gl.shaderSource(compositeShader, cS); + gl.compileShader(compositeShader); + + // Create the composite program + const compositeProgram = gl.createProgram(); + if (!compositeProgram) { + throw Error('can not create composite program'); + } + gl.attachShader(compositeProgram, vertexShader); + gl.attachShader(compositeProgram, compositeShader); + gl.linkProgram(compositeProgram); + + let blurProgram = null; + let blurVertexShader = null; + let blurFrag = null; + let blurUniforms = null; + + if (options.enableBlur) { + // Create blur shader if enabled + blurFrag = gl.createShader(gl.FRAGMENT_SHADER); + if (!blurFrag) { + throw Error('can not create blur shader'); + } + gl.shaderSource(blurFrag, blurFragmentShader); + gl.compileShader(blurFrag); + + // Get compile status and log errors if any + if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) { + const info = gl.getShaderInfoLog(blurFrag); + throw Error(`Failed to compile blur shader: ${info}`); + } + + // Create blur program + blurVertexShader = gl.createShader(gl.VERTEX_SHADER); + if (!blurVertexShader) { + throw Error('can not create blur vertex shader'); + } + gl.shaderSource(blurVertexShader, vs); + gl.compileShader(blurVertexShader); + + blurProgram = gl.createProgram(); + if (!blurProgram) { + throw Error('can not create blur program'); + } + gl.attachShader(blurProgram, blurVertexShader); + gl.attachShader(blurProgram, blurFrag); + gl.linkProgram(blurProgram); + + // Check blur program link status + if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) { + const info = gl.getProgramInfoLog(blurProgram); + throw Error(`Failed to link blur program: ${info}`); + } + + gl.useProgram(blurProgram); + blurUniforms = { + position: gl.getAttribLocation(blurProgram, 'position'), + texture: gl.getUniformLocation(blurProgram, 'u_texture'), + texelSize: gl.getUniformLocation(blurProgram, 'u_texelSize'), + direction: gl.getUniformLocation(blurProgram, 'u_direction'), + radius: gl.getUniformLocation(blurProgram, 'u_radius'), + }; + } + + return { + vertexShader, + compositeShader, + blurShader: blurFrag, + compositeProgram, + blurProgram, + attribLocations: { + position: gl.getAttribLocation(compositeProgram, 'position'), + }, + uniformLocations: { + mask: gl.getUniformLocation(compositeProgram, 'mask')!, + frame: gl.getUniformLocation(compositeProgram, 'frame')!, + background: gl.getUniformLocation(compositeProgram, 'background')!, + }, + blurUniforms, + }; +}; + +export function initTexture(gl: WebGL2RenderingContext, texIndex: number) { + const texRef = gl.TEXTURE0 + texIndex; + gl.activeTexture(texRef); + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.bindTexture(gl.TEXTURE_2D, texture); + + return texture; +} + +export function createFramebuffer( + gl: WebGL2RenderingContext, + texture: WebGLTexture, + width: number, + height: number, +) { + const framebuffer = gl.createFramebuffer(); + gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer); + + // Set the texture as the color attachment + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + + // Ensure texture dimensions match the provided width and height + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + + // Check if framebuffer is complete + const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER); + if (status !== gl.FRAMEBUFFER_COMPLETE) { + throw new Error('Framebuffer not complete'); + } + + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + return framebuffer; +} + +const createVertexBuffer = (gl: WebGL2RenderingContext) => { + if (!gl) { + return null; + } + const vertexBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]), + gl.STATIC_DRAW, + ); + return vertexBuffer; +}; + +export interface WebGLSetupOptions { + enableBlur?: boolean; + blurRadius?: number; +} + +export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = {}) => { + const gl = canvas.getContext('webgl2', { premultipliedAlpha: false }) as WebGL2RenderingContext; + + if (!gl) { + return undefined; + } + + gl.enable(gl.BLEND); + gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); + + const enableBlur = options.enableBlur ?? false; + const blurRadius = options.blurRadius ?? 5.0; + + const { + compositeProgram, + blurProgram, + attribLocations: { position: positionLocation }, + uniformLocations: { + mask: maskTextureLocation, + frame: frameTextureLocation, + background: bgTextureLocation, + }, + blurUniforms, + } = createShaderProgram(gl, { enableBlur }); + + const bgTexture = initTexture(gl, 0); + const frameTexture = initTexture(gl, 1); + const vertexBuffer = createVertexBuffer(gl); + + // Create additional textures and framebuffers for blur pass if enabled + let blurTextures: WebGLTexture[] = []; + let blurFramebuffers: WebGLFramebuffer[] = []; + + if (enableBlur) { + // Create two textures for blur passes (horizontal and vertical) + blurTextures.push(initTexture(gl, 3)); + blurTextures.push(initTexture(gl, 4)); + + // Create framebuffers for blur passes + blurFramebuffers.push(createFramebuffer(gl, blurTextures[0], canvas.width, canvas.height)); + blurFramebuffers.push(createFramebuffer(gl, blurTextures[1], canvas.width, canvas.height)); + } + + // Set up uniforms for the composite shader + gl.useProgram(compositeProgram); + gl.uniform1i(bgTextureLocation, 0); + gl.uniform1i(frameTextureLocation, 1); + gl.uniform1i(maskTextureLocation, 2); + + function applyBlur(sourceTexture: WebGLTexture, width: number, height: number) { + if (!enableBlur || !blurProgram || !blurUniforms) return sourceTexture; + + gl.useProgram(blurProgram); + + // Set common attributes + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.vertexAttribPointer(blurUniforms.position, 2, gl.FLOAT, false, 0, 0); + gl.enableVertexAttribArray(blurUniforms.position); + + const texelWidth = 1.0 / width; + const texelHeight = 1.0 / height; + + // First pass - horizontal blur + gl.bindFramebuffer(gl.FRAMEBUFFER, blurFramebuffers[0]); + gl.viewport(0, 0, width, height); + + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, sourceTexture); + gl.uniform1i(blurUniforms.texture, 0); + gl.uniform2f(blurUniforms.texelSize, texelWidth, texelHeight); + gl.uniform2f(blurUniforms.direction, 1.0, 0.0); // Horizontal + gl.uniform1f(blurUniforms.radius, blurRadius); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // Second pass - vertical blur + gl.bindFramebuffer(gl.FRAMEBUFFER, blurFramebuffers[1]); + gl.viewport(0, 0, width, height); + + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, blurTextures[0]); + gl.uniform1i(blurUniforms.texture, 0); + gl.uniform2f(blurUniforms.direction, 0.0, 1.0); // Vertical + + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // Reset framebuffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + return blurTextures[1]; + } + + function render(frame: VideoFrame, mask: MPMask) { + if (frame.codedWidth === 0 || mask.width === 0) { + return; + } + + const width = frame.displayWidth; + const height = frame.displayHeight; + + // Prepare frame texture + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, frameTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); + + // Apply blur if enabled + let backgroundTexture = bgTexture; + if (enableBlur) { + backgroundTexture = applyBlur(frameTexture, width, height); + } + + // Render the final composite + gl.viewport(0, 0, width, height); + gl.clearColor(1.0, 1.0, 1.0, 1.0); + gl.clear(gl.COLOR_BUFFER_BIT); + + gl.useProgram(compositeProgram); + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); + gl.enableVertexAttribArray(positionLocation); + + // Set background texture (either original or blurred) + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, backgroundTexture); + gl.uniform1i(bgTextureLocation, 0); + + // Set frame texture + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, frameTexture); + gl.uniform1i(frameTextureLocation, 1); + + // Set mask texture + const maskTexture = mask.getAsWebGLTexture(); + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, maskTexture); + gl.uniform1i(maskTextureLocation, 2); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + } + + return { render }; +}; From c90e338b91724a626b329d2afdd6a2182cca0444 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 17:08:51 +0200 Subject: [PATCH 04/12] working poc --- package.json | 2 +- pnpm-lock.yaml | 10 +-- src/transformers/BackgroundTransformer.ts | 8 +- src/transformers/VideoTransformer.ts | 2 + src/webgl/index.ts | 93 +++++++++++++++++++++-- 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 3ee5e92..c0e722c 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "src" ], "dependencies": { - "@mediapipe/tasks-vision": "0.10.21" + "@mediapipe/tasks-vision": "^0.10.22-rc.20250304" }, "peerDependencies": { "livekit-client": "^1.12.0 || ^2.1.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bedf1c4..1adc01f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@mediapipe/tasks-vision': - specifier: 0.10.21 - version: 0.10.21 + specifier: ^0.10.22-rc.20250304 + version: 0.10.22-rc.20250304 livekit-client: specifier: ^1.12.0 || ^2.1.0 version: 2.11.2 @@ -387,8 +387,8 @@ packages: '@mdn/browser-compat-data@5.2.55': resolution: {integrity: sha512-V5y5VhgXobwZl817zn+iAlCSTbXIXBMRHbL2WDyjJyMMgcHZoQTk6db1y3ZxBUo/H23MXgTKBo7bQ9S8aEfs2A==} - '@mediapipe/tasks-vision@0.10.21': - resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==} + '@mediapipe/tasks-vision@0.10.22-rc.20250304': + resolution: {integrity: sha512-dElxVXMFGthshfIj+qAVm8KE2jmNo2p8oXFib8WzEjb7GNaX/ClWBc8UJfoSZwjEMVrdHJ4YUfa7P3ifl6MIWw==} '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} @@ -2486,7 +2486,7 @@ snapshots: '@mdn/browser-compat-data@5.2.55': {} - '@mediapipe/tasks-vision@0.10.21': {} + '@mediapipe/tasks-vision@0.10.22-rc.20250304': {} '@nodelib/fs.scandir@2.1.5': dependencies: diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index 2f350ac..8d46326 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -33,8 +33,6 @@ export default class BackgroundProcessor extends VideoTransformer) { @@ -150,7 +150,7 @@ export default class BackgroundProcessor extends VideoTransformer canvasWidth * 1.5 || image.height > canvasHeight * 1.5; + + if (shouldResize) { + // Calculate new dimensions while maintaining aspect ratio + const aspectRatio = image.width / image.height; + let newWidth = canvasWidth; + let newHeight = canvasHeight; + + if (aspectRatio > 1) { + // Landscape orientation + newHeight = canvasWidth / aspectRatio; + } else { + // Portrait or square orientation + newWidth = canvasHeight * aspectRatio; + } + + // Resize the image using createImageBitmap with resize option + const resizedImage = await createImageBitmap(image, { + resizeWidth: Math.round(newWidth), + resizeHeight: Math.round(newHeight), + resizeQuality: 'medium', + }); + + // Store the resized image + customBackgroundImage = resizedImage; + + // Load the resized image into the texture + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, resizedImage); + } else { + // Use original image if it's already an appropriate size + customBackgroundImage = image; + + // Load the image into the texture + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); + } + } catch (error) { + console.error('Error resizing background image:', error); + // Fallback to original image on error + customBackgroundImage = image; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); + } + } + } + + function setBlurRadius(radius: number) { + blurRadius = radius; + setBackgroundImage(null); + } + + return { render, setBackgroundImage, setBlurRadius }; }; From 8662fa1ccd53fea6bae5f96672a9338539600c71 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 17:22:25 +0200 Subject: [PATCH 05/12] handle option updates --- src/transformers/BackgroundTransformer.ts | 5 +- src/transformers/VideoTransformer.ts | 2 - src/webgl/index.ts | 121 ++++++++++------------ 3 files changed, 57 insertions(+), 71 deletions(-) diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index 8d46326..4f42f0f 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -70,6 +70,9 @@ export default class BackgroundProcessor extends VideoTransformer { +const createShaderProgram = (gl: WebGL2RenderingContext) => { const vs = ` attribute vec2 position; varying vec2 texCoords; @@ -95,53 +91,50 @@ const createShaderProgram = (gl: WebGL2RenderingContext, options: ShaderProgramO let blurFrag = null; let blurUniforms = null; - if (options.enableBlur) { - // Create blur shader if enabled - blurFrag = gl.createShader(gl.FRAGMENT_SHADER); - if (!blurFrag) { - throw Error('can not create blur shader'); - } - gl.shaderSource(blurFrag, blurFragmentShader); - gl.compileShader(blurFrag); - - // Get compile status and log errors if any - if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) { - const info = gl.getShaderInfoLog(blurFrag); - throw Error(`Failed to compile blur shader: ${info}`); - } + // Create blur shader if enabled + blurFrag = gl.createShader(gl.FRAGMENT_SHADER); + if (!blurFrag) { + throw Error('can not create blur shader'); + } + gl.shaderSource(blurFrag, blurFragmentShader); + gl.compileShader(blurFrag); - // Create blur program - blurVertexShader = gl.createShader(gl.VERTEX_SHADER); - if (!blurVertexShader) { - throw Error('can not create blur vertex shader'); - } - gl.shaderSource(blurVertexShader, vs); - gl.compileShader(blurVertexShader); + // Get compile status and log errors if any + if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) { + const info = gl.getShaderInfoLog(blurFrag); + throw Error(`Failed to compile blur shader: ${info}`); + } - blurProgram = gl.createProgram(); - if (!blurProgram) { - throw Error('can not create blur program'); - } - gl.attachShader(blurProgram, blurVertexShader); - gl.attachShader(blurProgram, blurFrag); - gl.linkProgram(blurProgram); - - // Check blur program link status - if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) { - const info = gl.getProgramInfoLog(blurProgram); - throw Error(`Failed to link blur program: ${info}`); - } + // Create blur program + blurVertexShader = gl.createShader(gl.VERTEX_SHADER); + if (!blurVertexShader) { + throw Error('can not create blur vertex shader'); + } + gl.shaderSource(blurVertexShader, vs); + gl.compileShader(blurVertexShader); - gl.useProgram(blurProgram); - blurUniforms = { - position: gl.getAttribLocation(blurProgram, 'position'), - texture: gl.getUniformLocation(blurProgram, 'u_texture'), - texelSize: gl.getUniformLocation(blurProgram, 'u_texelSize'), - direction: gl.getUniformLocation(blurProgram, 'u_direction'), - radius: gl.getUniformLocation(blurProgram, 'u_radius'), - }; + blurProgram = gl.createProgram(); + if (!blurProgram) { + throw Error('can not create blur program'); + } + gl.attachShader(blurProgram, blurVertexShader); + gl.attachShader(blurProgram, blurFrag); + gl.linkProgram(blurProgram); + + // Check blur program link status + if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) { + const info = gl.getProgramInfoLog(blurProgram); + throw Error(`Failed to link blur program: ${info}`); } + blurUniforms = { + position: gl.getAttribLocation(blurProgram, 'position'), + texture: gl.getUniformLocation(blurProgram, 'u_texture'), + texelSize: gl.getUniformLocation(blurProgram, 'u_texelSize'), + direction: gl.getUniformLocation(blurProgram, 'u_direction'), + radius: gl.getUniformLocation(blurProgram, 'u_radius'), + }; + return { vertexShader, compositeShader, @@ -214,14 +207,11 @@ const createVertexBuffer = (gl: WebGL2RenderingContext) => { return vertexBuffer; }; -export interface WebGLSetupOptions { - enableBlur?: boolean; - blurRadius?: number; -} - -export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = {}) => { +export const setupWebGL = (canvas: OffscreenCanvas) => { const gl = canvas.getContext('webgl2', { premultipliedAlpha: false }) as WebGL2RenderingContext; + let blurRadius: number | null = null; + if (!gl) { return undefined; } @@ -229,9 +219,6 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = gl.enable(gl.BLEND); gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); - const enableBlur = options.enableBlur ?? false; - let blurRadius = options.blurRadius ?? 10.0; - const { compositeProgram, blurProgram, @@ -242,7 +229,7 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = background: bgTextureLocation, }, blurUniforms, - } = createShaderProgram(gl, { enableBlur }); + } = createShaderProgram(gl); const bgTexture = initTexture(gl, 0); const frameTexture = initTexture(gl, 1); @@ -252,15 +239,13 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = let blurTextures: WebGLTexture[] = []; let blurFramebuffers: WebGLFramebuffer[] = []; - if (enableBlur) { - // Create two textures for blur passes (horizontal and vertical) - blurTextures.push(initTexture(gl, 3)); - blurTextures.push(initTexture(gl, 4)); + // Create two textures for blur passes (horizontal and vertical) + blurTextures.push(initTexture(gl, 3)); + blurTextures.push(initTexture(gl, 4)); - // Create framebuffers for blur passes - blurFramebuffers.push(createFramebuffer(gl, blurTextures[0], canvas.width, canvas.height)); - blurFramebuffers.push(createFramebuffer(gl, blurTextures[1], canvas.width, canvas.height)); - } + // Create framebuffers for blur passes + blurFramebuffers.push(createFramebuffer(gl, blurTextures[0], canvas.width, canvas.height)); + blurFramebuffers.push(createFramebuffer(gl, blurTextures[1], canvas.width, canvas.height)); // Set up uniforms for the composite shader gl.useProgram(compositeProgram); @@ -272,7 +257,7 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = let customBackgroundImage: ImageBitmap | null = null; function applyBlur(sourceTexture: WebGLTexture, width: number, height: number) { - if (!enableBlur || !blurProgram || !blurUniforms) return sourceTexture; + if (!blurRadius || !blurProgram || !blurUniforms) return sourceTexture; gl.useProgram(blurProgram); @@ -336,7 +321,7 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = gl.bindTexture(gl.TEXTURE_2D, bgTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage); backgroundTexture = bgTexture; - } else if (enableBlur) { + } else if (blurRadius) { // Otherwise, if blur is enabled, apply blur effect to the frame backgroundTexture = applyBlur(frameTexture, width, height); } @@ -437,7 +422,7 @@ export const setupWebGL = (canvas: OffscreenCanvas, options: WebGLSetupOptions = } } - function setBlurRadius(radius: number) { + function setBlurRadius(radius: number | null) { blurRadius = radius; setBackgroundImage(null); } From a7d5eea8a4cf81eb2f1f589626d8e9719110fc0d Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 17:37:07 +0200 Subject: [PATCH 06/12] cleanup resources --- src/transformers/BackgroundTransformer.ts | 104 ++-------------------- src/webgl/index.ts | 65 +++++++++++++- 2 files changed, 67 insertions(+), 102 deletions(-) diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index 4f42f0f..d5267df 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -57,6 +57,7 @@ export default class BackgroundProcessor extends VideoTransformer 0) { - // this.ctx.globalCompositeOperation = 'copy'; - - // this.ctx.putImageData( - // maskToImageData( - // this.segmentationResults.categoryMask, - // this.segmentationResults.categoryMask.width, - // this.segmentationResults.categoryMask.height, - // ), - // 0, - // 0, - // ); - // this.ctx.filter = 'none'; - // this.ctx.globalCompositeOperation = 'source-in'; - // if (this.backgroundImage) { - // this.ctx.drawImage( - // this.backgroundImage, - // 0, - // 0, - // this.backgroundImage.width, - // this.backgroundImage.height, - // 0, - // 0, - // this.canvas.width, - // this.canvas.height, - // ); - // } else { - // this.ctx.fillStyle = '#00FF00'; - // this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height); - // } - - // this.ctx.globalCompositeOperation = 'destination-over'; - // } - // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - // } - - // async blurBackground(frame: VideoFrame) { - // if ( - // !this.ctx || - // !this.canvas || - // !this.segmentationResults?.categoryMask?.canvas || - // !this.inputVideo - // ) { - // return; - // } - - // this.ctx.save(); - // this.ctx.globalCompositeOperation = 'copy'; - - // if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) { - // this.ctx.putImageData( - // maskToImageData( - // this.segmentationResults.categoryMask, - // this.segmentationResults.categoryMask.width, - // this.segmentationResults.categoryMask.height, - // ), - // 0, - // 0, - // ); - // this.ctx.filter = 'none'; - // this.ctx.globalCompositeOperation = 'source-out'; - // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - // this.ctx.globalCompositeOperation = 'destination-over'; - // this.ctx.filter = `blur(${this.blurRadius}px)`; - // this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - // this.ctx.restore(); - // } - // } } - -// function maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData { -// const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4); -// const result = mask.getAsUint8Array(); -// for (let i = 0; i < result.length; i += 1) { -// const offset = i * 4; -// dataArray[offset] = result[i]; -// dataArray[offset + 1] = result[i]; -// dataArray[offset + 2] = result[i]; -// dataArray[offset + 3] = result[i]; -// } -// return new ImageData(dataArray, videoWidth, videoHeight); -// } diff --git a/src/webgl/index.ts b/src/webgl/index.ts index b62799f..61b4b6c 100644 --- a/src/webgl/index.ts +++ b/src/webgl/index.ts @@ -257,7 +257,7 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { let customBackgroundImage: ImageBitmap | null = null; function applyBlur(sourceTexture: WebGLTexture, width: number, height: number) { - if (!blurRadius || !blurProgram || !blurUniforms) return sourceTexture; + if (!blurRadius || !blurProgram || !blurUniforms) return bgTexture; gl.useProgram(blurProgram); @@ -351,8 +351,9 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, maskTexture); gl.uniform1i(maskTextureLocation, 2); - gl.drawArrays(gl.TRIANGLES, 0, 6); + + mask.close(); } /** @@ -419,6 +420,16 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { gl.bindTexture(gl.TEXTURE_2D, bgTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); } + } else { + // set the background texture to an empty 2x2 image + const emptyImage = new ImageData(2, 2); + emptyImage.data[0] = 0; + emptyImage.data[1] = 0; + emptyImage.data[2] = 0; + emptyImage.data[3] = 0; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, emptyImage); } } @@ -427,5 +438,53 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { setBackgroundImage(null); } - return { render, setBackgroundImage, setBlurRadius }; + /** + * Cleans up all WebGL resources to prevent memory leaks + */ + function cleanup() { + // Clean up shader programs + if (compositeProgram) { + gl.deleteProgram(compositeProgram); + } + + if (blurProgram) { + gl.deleteProgram(blurProgram); + } + + // Clean up textures + if (bgTexture) { + gl.deleteTexture(bgTexture); + } + + if (frameTexture) { + gl.deleteTexture(frameTexture); + } + + // Clean up blur textures + for (const texture of blurTextures) { + gl.deleteTexture(texture); + } + + // Clean up framebuffers + for (const framebuffer of blurFramebuffers) { + gl.deleteFramebuffer(framebuffer); + } + + // Clean up vertex buffer + if (vertexBuffer) { + gl.deleteBuffer(vertexBuffer); + } + + // Release any ImageBitmap resources + if (customBackgroundImage) { + customBackgroundImage.close(); + customBackgroundImage = null; + } + + // Clear arrays + blurTextures = []; + blurFramebuffers = []; + } + + return { render, setBackgroundImage, setBlurRadius, cleanup }; }; From 895ff0f2fea14002fb2a74e5e1de9e1b97978e05 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 17:37:56 +0200 Subject: [PATCH 07/12] cleanup code --- src/utils.ts | 368 --------------------------------------------------- src/webgl.ts | 219 ------------------------------ 2 files changed, 587 deletions(-) delete mode 100644 src/webgl.ts diff --git a/src/utils.ts b/src/utils.ts index 3c55b7c..ec52103 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,3 @@ -import { MPMask } from '@mediapipe/tasks-vision'; - /* eslint-disable @typescript-eslint/naming-convention */ export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined'; export const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined'; @@ -25,369 +23,3 @@ export async function waitForTrackResolution(track: MediaStreamTrack) { } return { width: undefined, height: undefined }; } - -export type WebGLRenderer = { - gl: WebGLRenderingContext; - render: (videoFrame: VideoFrame, radius: number, mask: TexImageSource) => void; - cleanup: () => void; -}; - -export function initWebGL(canvas: OffscreenCanvas): WebGLRenderer { - const gl = canvas.getContext('webgl'); - if (!gl) throw new Error('WebGL not supported'); - - const vsSource = ` - attribute vec2 a_position; - attribute vec2 a_texCoord; - varying vec2 v_texCoord; - void main() { - gl_Position = vec4(a_position, 0, 1); - v_texCoord = a_texCoord; - } - `; - - const fsSource = ` - precision mediump float; - varying vec2 v_texCoord; - uniform sampler2D u_texture; - void main() { - gl_FragColor = texture2D(u_texture, v_texCoord); - } - `; - - const createShader = (type: number, source: string): WebGLShader => { - const shader = gl.createShader(type); - if (!shader) throw new Error('Failed to create shader'); - gl.shaderSource(shader, source); - gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - throw new Error(gl.getShaderInfoLog(shader) || 'Unknown shader error'); - } - return shader; - }; - - const vertexShader = createShader(gl.VERTEX_SHADER, vsSource); - const fragmentShader = createShader(gl.FRAGMENT_SHADER, fsSource); - - const program = gl.createProgram(); - if (!program) throw new Error('Failed to create program'); - gl.attachShader(program, vertexShader); - gl.attachShader(program, fragmentShader); - gl.linkProgram(program); - if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - throw new Error(gl.getProgramInfoLog(program) || 'Program linking failed'); - } - gl.useProgram(program); - - // Position buffer - const positionBuffer = gl.createBuffer(); - gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); - const positions = new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]); - gl.bufferData(gl.ARRAY_BUFFER, positions, gl.STATIC_DRAW); - - const posLoc = gl.getAttribLocation(program, 'a_position'); - gl.enableVertexAttribArray(posLoc); - gl.vertexAttribPointer(posLoc, 2, gl.FLOAT, false, 0, 0); - - // Texture coord buffer - const texCoordBuffer = gl.createBuffer(); - gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); - const texCoords = new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]); - gl.bufferData(gl.ARRAY_BUFFER, texCoords, gl.STATIC_DRAW); - - const texLoc = gl.getAttribLocation(program, 'a_texCoord'); - gl.enableVertexAttribArray(texLoc); - gl.vertexAttribPointer(texLoc, 2, gl.FLOAT, false, 0, 0); - - // Texture setup - const texture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - - const render = (videoFrame: VideoFrame): void => { - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - gl.viewport(0, 0, canvas.width, canvas.height); - gl.clear(gl.COLOR_BUFFER_BIT); - gl.drawArrays(gl.TRIANGLES, 0, 6); - }; - - const cleanup = (): void => { - gl.deleteTexture(texture); - gl.deleteBuffer(positionBuffer); - gl.deleteBuffer(texCoordBuffer); - gl.deleteProgram(program); - gl.deleteShader(vertexShader); - gl.deleteShader(fragmentShader); - }; - - return { gl, render, cleanup }; -} - -type BlurPipeline = { - render: (frame: VideoFrame, radius: number, mask: MPMask) => void; - cleanup: () => void; -}; - -export function createBlurPipeline(canvas: OffscreenCanvas): BlurPipeline { - const gl = canvas.getContext('webgl')!; - if (!gl) throw new Error('WebGL not supported'); - - const baseVertexShader = ` - attribute vec2 a_position; - attribute vec2 a_texCoord; - varying vec2 v_texCoord; - uniform float u_flipY; - - void main() { - gl_Position = vec4(a_position, 0.0, 1.0); - v_texCoord = vec2(a_texCoord.x, u_flipY == 1.0 ? 1.0 - a_texCoord.y : a_texCoord.y); - } - `; - - const blurFragmentShader = ` - precision mediump float; - varying vec2 v_texCoord; - uniform sampler2D u_texture; - uniform vec2 u_texelSize; - uniform vec2 u_direction; - uniform float u_radius; - - void main() { - float sigma = u_radius; - float twoSigmaSq = 2.0 * sigma * sigma; - float totalWeight = 0.0; - vec3 result = vec3(0.0); - const int MAX_SAMPLES = 16; - int radius = int(min(float(MAX_SAMPLES), ceil(u_radius))); - - for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) { - float offset = float(i); - if (abs(offset) > float(radius)) continue; - float weight = exp(-(offset * offset) / twoSigmaSq); - vec2 sampleCoord = v_texCoord + u_direction * u_texelSize * offset; - result += texture2D(u_texture, sampleCoord).rgb * weight; - totalWeight += weight; - } - - gl_FragColor = vec4(result / totalWeight, 1.0); - } - `; - - const compositeFragmentShader = ` - precision mediump float; - varying vec2 v_texCoord; - uniform sampler2D u_original; - uniform sampler2D u_blurred; - uniform sampler2D u_mask; - - void main() { - vec4 orig = texture2D(u_original, v_texCoord); - vec4 blur = texture2D(u_blurred, v_texCoord); - vec4 mask = texture2D(u_mask, v_texCoord); - gl_FragColor = vec4(mask.r * 10.0, mask.g * 10.0, mask.b * 10.0, orig.a); - } -`; - - const compositeProgram = createProgram(baseVertexShader, compositeFragmentShader); - - // --- Compile - function compileShader(type: number, source: string): WebGLShader { - const shader = gl.createShader(type)!; - gl.shaderSource(shader, source); - gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - throw new Error(gl.getShaderInfoLog(shader)!); - } - return shader; - } - - function createProgram(vsSrc: string, fsSrc: string): WebGLProgram { - const vs = compileShader(gl.VERTEX_SHADER, vsSrc); - const fs = compileShader(gl.FRAGMENT_SHADER, fsSrc); - const program = gl.createProgram()!; - gl.attachShader(program, vs); - gl.attachShader(program, fs); - gl.linkProgram(program); - if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - throw new Error(gl.getProgramInfoLog(program)!); - } - return program; - } - - const program = createProgram(baseVertexShader, blurFragmentShader); - - const quad = new Float32Array([ - -1, -1, 0, 1, 1, -1, 1, 1, -1, 1, 0, 0, -1, 1, 0, 0, 1, -1, 1, 1, 1, 1, 1, 0, - ]); - - const quadBuffer = gl.createBuffer()!; - gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); - gl.bufferData(gl.ARRAY_BUFFER, quad, gl.STATIC_DRAW); - - const positionLoc = gl.getAttribLocation(program, 'a_position'); - const texCoordLoc = gl.getAttribLocation(program, 'a_texCoord'); - - const u_texture = gl.getUniformLocation(program, 'u_texture'); - const u_texelSize = gl.getUniformLocation(program, 'u_texelSize'); - const u_direction = gl.getUniformLocation(program, 'u_direction'); - const u_radius = gl.getUniformLocation(program, 'u_radius'); - const u_flipY = gl.getUniformLocation(program, 'u_flipY'); - - const texture = gl.createTexture()!; - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - - const framebuffer = gl.createFramebuffer()!; - gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer); - gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - - // Second texture and framebuffer for vertical blur pass - const blurTexture = gl.createTexture()!; - gl.bindTexture(gl.TEXTURE_2D, blurTexture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.RGBA, - canvas.width, - canvas.height, - 0, - gl.RGBA, - gl.UNSIGNED_BYTE, - null, - ); - - // const maskTexture = gl.createTexture()!; - // gl.bindTexture(gl.TEXTURE_2D, maskTexture); - // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - // gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - - const blurFBO = gl.createFramebuffer()!; - gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); - gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, blurTexture, 0); - - // === Render Function === - function render(frame: VideoFrame, radius: number, mask: MPMask) { - if (!mask.canvas) { - console.warn('MPMask does not have a canvas, skipping render'); - return; - } - - const maskContext = mask.canvas.getContext('webgl') as WebGLRenderingContext | null; - if (!maskContext) { - console.warn('MPMask canvas does not have WebGL context, skipping render'); - return; - } - - // Bind the framebuffer for horizontal blur - gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer); - gl.viewport(0, 0, canvas.width, canvas.height); - - // Use the blur program for horizontal pass - gl.useProgram(program); - - // Set up vertex attributes - gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer); - gl.enableVertexAttribArray(positionLoc); - gl.vertexAttribPointer(positionLoc, 2, gl.FLOAT, false, 16, 0); - gl.enableVertexAttribArray(texCoordLoc); - gl.vertexAttribPointer(texCoordLoc, 2, gl.FLOAT, false, 16, 8); - - // Set uniforms for horizontal blur - gl.uniform1i(u_texture, 0); - gl.uniform2f(u_texelSize, 1.0 / canvas.width, 1.0 / canvas.height); - gl.uniform2f(u_direction, 1.0, 0.0); - gl.uniform1f(u_radius, radius); - gl.uniform1f(u_flipY, 1.0); - - // Bind and upload the frame to texture - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); - - // Draw horizontal blur - gl.drawArrays(gl.TRIANGLES, 0, 6); - - // Bind the framebuffer for vertical blur - gl.bindFramebuffer(gl.FRAMEBUFFER, blurFBO); - - // Set uniforms for vertical blur - gl.uniform2f(u_direction, 0.0, 1.0); - - // Bind the horizontally blurred texture - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - - // Draw vertical blur - gl.drawArrays(gl.TRIANGLES, 0, 6); - - // Now use the composite program to blend with mask - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.useProgram(compositeProgram); - - // Set up vertex attributes for composite - gl.enableVertexAttribArray(gl.getAttribLocation(compositeProgram, 'a_position')); - gl.vertexAttribPointer( - gl.getAttribLocation(compositeProgram, 'a_position'), - 2, - gl.FLOAT, - false, - 16, - 0, - ); - gl.enableVertexAttribArray(gl.getAttribLocation(compositeProgram, 'a_texCoord')); - gl.vertexAttribPointer( - gl.getAttribLocation(compositeProgram, 'a_texCoord'), - 2, - gl.FLOAT, - false, - 16, - 8, - ); - - // Set uniforms for composite - gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_original'), 0); - gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_blurred'), 1); - gl.uniform1i(gl.getUniformLocation(compositeProgram, 'u_mask'), 2); - - // Bind textures - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.activeTexture(gl.TEXTURE1); - gl.bindTexture(gl.TEXTURE_2D, blurTexture); - gl.activeTexture(gl.TEXTURE2); - // The mask texture is already bound to mask.canvas, so we can use it directly - const maskTexture = maskContext.getParameter(maskContext.TEXTURE_BINDING_2D); - gl.bindTexture(gl.TEXTURE_2D, maskTexture); - - // Draw final composite - gl.drawArrays(gl.TRIANGLES, 0, 6); - } - - function cleanup() { - gl.deleteBuffer(quadBuffer); - gl.deleteTexture(texture); - gl.deleteFramebuffer(framebuffer); - gl.deleteTexture(blurTexture); - gl.deleteFramebuffer(blurFBO); - } - - return { render, cleanup }; -} diff --git a/src/webgl.ts b/src/webgl.ts deleted file mode 100644 index 3ae32d5..0000000 --- a/src/webgl.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { FilesetResolver, ImageSegmenter, ImageSegmenterResult } from '@mediapipe/tasks-vision'; - -let videoTexture: WebGLTexture, bgTexture: WebGLTexture; - -let bgImage = 'https://videos.electroteque.org/textures/virtualbg.jpg'; - -function initBkgnd( - gl: WebGL2RenderingContext, - bgImage: string, - bgTextureLocation: WebGLUniformLocation, -) { - bgTexture = initTexture(gl, 0); - gl.uniform1i(bgTextureLocation, 0); - - if (bgImage) { - const img = new Image(); - img.crossOrigin = 'anonymous'; - img.onload = () => { - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, bgTexture); - //resizeForTexture(img, img.width, img.height); - console.log('FILL BACKGROUND'); - fillBackgroundImage(gl, img); - }; - img.src = bgImage; - } else { - // Fill with black background - gl.clearColor(0.0, 0.0, 0.0, 1.0); - gl.clear(gl.COLOR_BUFFER_BIT); - } -} - -function fillBackgroundImage(gl: WebGL2RenderingContext, img: HTMLImageElement) { - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, img); - // gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); -} - -const createShaderProgram = (gl: WebGL2RenderingContext) => { - const vs = ` - attribute vec2 position; - varying vec2 texCoords; - - void main() { - texCoords = (position + 1.0) / 2.0; - texCoords.y = 1.0 - texCoords.y; - gl_Position = vec4(position, 0, 1.0); - } - `; - - const fs = ` - precision highp float; - varying vec2 texCoords; - uniform sampler2D background; - uniform sampler2D frame; - uniform sampler2D mask; - void main() { - vec4 maskTex = texture2D(mask, texCoords); - vec4 frameTex = texture2D(frame, texCoords); - vec4 bgTex = texture2D(background, texCoords); - - - float a = maskTex.r; - - gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), a); - //gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - a); - - } - `; - - const vertexShader = gl.createShader(gl.VERTEX_SHADER); - if (!vertexShader) { - throw Error('can not create vertex shader'); - } - gl.shaderSource(vertexShader, vs); - gl.compileShader(vertexShader); - - // Create our fragment shader - const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER); - if (!fragmentShader) { - throw Error('can not create fragment shader'); - } - gl.shaderSource(fragmentShader, fs); - gl.compileShader(fragmentShader); - - // Create our program - const program = gl.createProgram(); - if (!program) { - throw Error('can not create program'); - } - gl.attachShader(program, vertexShader); - gl.attachShader(program, fragmentShader); - gl.linkProgram(program); - - //bgTexture = initTexture(gl, 0); - initBkgnd(gl, bgImage, gl.getUniformLocation(program, 'background')!); - videoTexture = initTexture(gl, 1); - - return { - vertexShader, - fragmentShader, - shaderProgram: program, - attribLocations: { - position: gl.getAttribLocation(program, 'position'), - }, - uniformLocations: { - mask: gl.getUniformLocation(program, 'mask')!, - frame: gl.getUniformLocation(program, 'frame')!, - background: gl.getUniformLocation(program, 'background')!, - }, - }; -}; - -const createVertexBuffer = (gl: WebGL2RenderingContext) => { - if (!gl) { - return null; - } - const vertexBuffer = gl.createBuffer(); - gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); - gl.bufferData( - gl.ARRAY_BUFFER, - new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]), - gl.STATIC_DRAW, - ); - return vertexBuffer; -}; - -function initTexture(gl: WebGL2RenderingContext, texIndex: number) { - const texRef = gl.TEXTURE0 + texIndex; - gl.activeTexture(gl.TEXTURE0 + texIndex); - const texture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - //gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - - gl.bindTexture(gl.TEXTURE_2D, texture); - - return texture; -} - -function createCopyTextureToCanvas(canvas: HTMLCanvasElement) { - const gl = canvas.getContext('webgl2', { premultipliedAlpha: false }) as WebGL2RenderingContext; - - gl.enable(gl.BLEND); - gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); - - if (!gl) { - return undefined; - } - const { - shaderProgram, - attribLocations: { position: positionLocation }, - uniformLocations: { - mask: maskTextureLocation, - frame: frameTextureLocation, - background: bgTextureLocation, - }, - } = createShaderProgram(gl); - const vertexBuffer = createVertexBuffer(gl); - - gl.uniform1i(bgTextureLocation, 0); - gl.uniform1i(frameTextureLocation, 1); - gl.uniform1i(maskTextureLocation, 2); - - return (mask: { getAsWebGLTexture: () => WebGLTexture }) => { - //gl.viewport(0, 0, canvas.width, canvas.height) - gl.clearColor(1.0, 1.0, 1.0, 1.0); - gl.useProgram(shaderProgram); - gl.clear(gl.COLOR_BUFFER_BIT); - gl.viewport(0, 0, video.videoWidth, video.videoHeight); - //gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); - - const texture = mask.getAsWebGLTexture(); - gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); - gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); - gl.enableVertexAttribArray(positionLocation); - - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, bgTexture); - //gl.uniform1i(bgTextureLocation, 0) - - gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.uniform1i(maskTextureLocation, 2); - - gl.activeTexture(gl.TEXTURE1); - gl.bindTexture(gl.TEXTURE_2D, videoTexture); - gl.uniform1i(frameTextureLocation, 1); - - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video); - - gl.drawArrays(gl.TRIANGLES, 0, 6); - - return createImageBitmap(canvas); - }; -} - -const tasksCanvas = new OffscreenCanvas(1, 1); -const createImageSegmenter = async () => { - const audio = await FilesetResolver.forVisionTasks( - 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm', - ); - - imageSegmenter = await ImageSegmenter.createFromOptions(audio, { - baseOptions: { - modelAssetPath: - //"https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/latest/deeplab_v3.tflite", - 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', - delegate: 'GPU', - }, - canvas: tasksCanvas, - runningMode: 'VIDEO', - outputConfidenceMasks: true, - }); -}; -createImageSegmenter(); From 0e2ef5b6cd9fbd2227eea69fedf050e9cf1e56d9 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 18:00:04 +0200 Subject: [PATCH 08/12] proper cleanup --- src/transformers/BackgroundTransformer.ts | 1 - src/transformers/VideoTransformer.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index d5267df..69ff278 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -79,7 +79,6 @@ export default class BackgroundProcessor extends VideoTransformer Date: Mon, 14 Apr 2025 18:41:47 +0200 Subject: [PATCH 09/12] crop background image --- src/webgl/index.ts | 77 +++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/src/webgl/index.ts b/src/webgl/index.ts index 61b4b6c..9e266ad 100644 --- a/src/webgl/index.ts +++ b/src/webgl/index.ts @@ -366,54 +366,47 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { if (image) { try { - // Determine appropriate size to avoid performance issues // Get current canvas dimensions const canvasWidth = canvas.width; const canvasHeight = canvas.height; - // Only resize if the image is significantly larger than the canvas - // A reasonable threshold is 1.5x the canvas size - const shouldResize = image.width > canvasWidth * 1.5 || image.height > canvasHeight * 1.5; - - if (shouldResize) { - // Calculate new dimensions while maintaining aspect ratio - const aspectRatio = image.width / image.height; - let newWidth = canvasWidth; - let newHeight = canvasHeight; - - if (aspectRatio > 1) { - // Landscape orientation - newHeight = canvasWidth / aspectRatio; - } else { - // Portrait or square orientation - newWidth = canvasHeight * aspectRatio; - } - - // Resize the image using createImageBitmap with resize option - const resizedImage = await createImageBitmap(image, { - resizeWidth: Math.round(newWidth), - resizeHeight: Math.round(newHeight), - resizeQuality: 'medium', - }); - - // Store the resized image - customBackgroundImage = resizedImage; - - // Load the resized image into the texture - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, bgTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, resizedImage); - } else { - // Use original image if it's already an appropriate size - customBackgroundImage = image; - - // Load the image into the texture - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, bgTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); + // Calculate dimensions and crop for "cover" mode + const imgAspect = image.width / image.height; + const canvasAspect = canvasWidth / canvasHeight; + + let sx = 0; + let sy = 0; + let sWidth = image.width; + let sHeight = image.height; + + // For cover mode, we need to crop some parts of the image + // to ensure it covers the canvas while maintaining aspect ratio + if (imgAspect > canvasAspect) { + // Image is wider than canvas - crop the sides + sWidth = Math.round(image.height * canvasAspect); + sx = Math.round((image.width - sWidth) / 2); // Center the crop horizontally + } else if (imgAspect < canvasAspect) { + // Image is taller than canvas - crop the top/bottom + sHeight = Math.round(image.width / canvasAspect); + sy = Math.round((image.height - sHeight) / 2); // Center the crop vertically } + + // Create a new ImageBitmap with the cropped portion + const croppedImage = await createImageBitmap(image, sx, sy, sWidth, sHeight, { + resizeWidth: canvasWidth, + resizeHeight: canvasHeight, + resizeQuality: 'medium', + }); + + // Store the cropped and resized image + customBackgroundImage = croppedImage; + + // Load the image into the texture + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, bgTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, croppedImage); } catch (error) { - console.error('Error resizing background image:', error); + console.error('Error processing background image:', error); // Fallback to original image on error customBackgroundImage = image; gl.activeTexture(gl.TEXTURE0); From cd91c3e2517fa1307d3c6057c1638ecba2fcdbd9 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 19:14:07 +0200 Subject: [PATCH 10/12] cleanup --- src/webgl/index.ts | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/webgl/index.ts b/src/webgl/index.ts index 9e266ad..70d72ba 100644 --- a/src/webgl/index.ts +++ b/src/webgl/index.ts @@ -56,7 +56,6 @@ const createShaderProgram = (gl: WebGL2RenderingContext) => { float a = maskTex.r; - //gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), a); gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - a); } @@ -235,17 +234,17 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { const frameTexture = initTexture(gl, 1); const vertexBuffer = createVertexBuffer(gl); - // Create additional textures and framebuffers for blur pass if enabled - let blurTextures: WebGLTexture[] = []; - let blurFramebuffers: WebGLFramebuffer[] = []; + // Create additional textures and framebuffers for processing + let processTextures: WebGLTexture[] = []; + let processFramebuffers: WebGLFramebuffer[] = []; - // Create two textures for blur passes (horizontal and vertical) - blurTextures.push(initTexture(gl, 3)); - blurTextures.push(initTexture(gl, 4)); + // Create textures for processing (blur) + processTextures.push(initTexture(gl, 3)); + processTextures.push(initTexture(gl, 4)); - // Create framebuffers for blur passes - blurFramebuffers.push(createFramebuffer(gl, blurTextures[0], canvas.width, canvas.height)); - blurFramebuffers.push(createFramebuffer(gl, blurTextures[1], canvas.width, canvas.height)); + // Create framebuffers for processing + processFramebuffers.push(createFramebuffer(gl, processTextures[0], canvas.width, canvas.height)); + processFramebuffers.push(createFramebuffer(gl, processTextures[1], canvas.width, canvas.height)); // Set up uniforms for the composite shader gl.useProgram(compositeProgram); @@ -270,7 +269,7 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { const texelHeight = 1.0 / height; // First pass - horizontal blur - gl.bindFramebuffer(gl.FRAMEBUFFER, blurFramebuffers[0]); + gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[0]); gl.viewport(0, 0, width, height); gl.activeTexture(gl.TEXTURE0); @@ -283,11 +282,11 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { gl.drawArrays(gl.TRIANGLES, 0, 6); // Second pass - vertical blur - gl.bindFramebuffer(gl.FRAMEBUFFER, blurFramebuffers[1]); + gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[1]); gl.viewport(0, 0, width, height); gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, blurTextures[0]); + gl.bindTexture(gl.TEXTURE_2D, processTextures[0]); gl.uniform1i(blurUniforms.texture, 0); gl.uniform2f(blurUniforms.direction, 0.0, 1.0); // Vertical @@ -296,7 +295,7 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { // Reset framebuffer gl.bindFramebuffer(gl.FRAMEBUFFER, null); - return blurTextures[1]; + return processTextures[1]; } function render(frame: VideoFrame, mask: MPMask) { @@ -326,6 +325,9 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { backgroundTexture = applyBlur(frameTexture, width, height); } + // Get the mask texture + const maskTexture = mask.getAsWebGLTexture(); + // Render the final composite gl.viewport(0, 0, width, height); gl.clearColor(1.0, 1.0, 1.0, 1.0); @@ -347,7 +349,6 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { gl.uniform1i(frameTextureLocation, 1); // Set mask texture - const maskTexture = mask.getAsWebGLTexture(); gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, maskTexture); gl.uniform1i(maskTextureLocation, 2); @@ -431,9 +432,6 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { setBackgroundImage(null); } - /** - * Cleans up all WebGL resources to prevent memory leaks - */ function cleanup() { // Clean up shader programs if (compositeProgram) { @@ -453,13 +451,13 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { gl.deleteTexture(frameTexture); } - // Clean up blur textures - for (const texture of blurTextures) { + // Clean up process textures + for (const texture of processTextures) { gl.deleteTexture(texture); } // Clean up framebuffers - for (const framebuffer of blurFramebuffers) { + for (const framebuffer of processFramebuffers) { gl.deleteFramebuffer(framebuffer); } @@ -475,8 +473,8 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { } // Clear arrays - blurTextures = []; - blurFramebuffers = []; + processTextures = []; + processFramebuffers = []; } return { render, setBackgroundImage, setBlurRadius, cleanup }; From ad69a9747c86cf41f00eb60693b6fec249fac6db Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 19:14:35 +0200 Subject: [PATCH 11/12] Create wet-pugs-sip.md --- .changeset/wet-pugs-sip.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/wet-pugs-sip.md diff --git a/.changeset/wet-pugs-sip.md b/.changeset/wet-pugs-sip.md new file mode 100644 index 0000000..1bc4a4f --- /dev/null +++ b/.changeset/wet-pugs-sip.md @@ -0,0 +1,5 @@ +--- +"@livekit/track-processors": minor +--- + +Use webGL for video processors From 4b8bbe32f470596db90241ef14d6fdc69763a813 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 14 Apr 2025 19:19:05 +0200 Subject: [PATCH 12/12] more cleanup --- src/webgl/index.ts | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/src/webgl/index.ts b/src/webgl/index.ts index 70d72ba..bd90a3e 100644 --- a/src/webgl/index.ts +++ b/src/webgl/index.ts @@ -433,46 +433,23 @@ export const setupWebGL = (canvas: OffscreenCanvas) => { } function cleanup() { - // Clean up shader programs - if (compositeProgram) { - gl.deleteProgram(compositeProgram); - } - - if (blurProgram) { - gl.deleteProgram(blurProgram); - } - - // Clean up textures - if (bgTexture) { - gl.deleteTexture(bgTexture); - } - - if (frameTexture) { - gl.deleteTexture(frameTexture); - } - - // Clean up process textures + gl.deleteProgram(compositeProgram); + gl.deleteProgram(blurProgram); + gl.deleteTexture(bgTexture); + gl.deleteTexture(frameTexture); for (const texture of processTextures) { gl.deleteTexture(texture); } - - // Clean up framebuffers for (const framebuffer of processFramebuffers) { gl.deleteFramebuffer(framebuffer); } - - // Clean up vertex buffer - if (vertexBuffer) { - gl.deleteBuffer(vertexBuffer); - } + gl.deleteBuffer(vertexBuffer); // Release any ImageBitmap resources if (customBackgroundImage) { customBackgroundImage.close(); customBackgroundImage = null; } - - // Clear arrays processTextures = []; processFramebuffers = []; }