Offscreencanvas

packages/gl/Dockerfile

@@ -1,18 +1,17 @@
 FROM mcr.microsoft.com/playwright:v1.49.1
-RUN npm install -g corepack@latest
-RUN corepack enable
+RUN npm install -g pnpm@10
 
 WORKDIR /app
 
-COPY pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY --from=workspace pnpm-lock.yaml pnpm-workspace.yaml ./
 
-COPY packages/gl/package.json ./packages/gl/
-COPY packages/helpers/package.json ./packages/helpers/
+COPY package.json ./packages/gl/
+COPY --from=helpers package.json ./packages/helpers/
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install
 
-COPY packages/gl ./packages/gl
-COPY packages/helpers ./packages/helpers
+COPY . ./packages/gl
+COPY --from=helpers . ./packages/helpers
 
 WORKDIR /app/packages/gl
 

packages/gl/compose.yml

@@ -1,8 +1,11 @@
 services:
   test:
     build:
-      context: ../..
-      dockerfile: packages/gl/Dockerfile
+      context: .
+      dockerfile: Dockerfile
+      additional_contexts:
+        workspace: ../..
+        helpers: ../helpers
     ports:
       - "9323:9323"
     volumes:

packages/gl/playground/src/pages/core/offscreencanvas.astro

@@ -0,0 +1,33 @@
+---
+import Layout from "../../layouts/Layout.astro";
+---
+
+<script>
+  import { onResize } from "@radiance/helpers";
+  import { incrementRenderCount } from "../../components/renderCount";
+
+  const canvas = document.querySelector("canvas")!;
+  const offscreenCanvas = canvas.transferControlToOffscreen();
+
+  const worker = new Worker(new URL("../../workers/offscreenGradient.worker.ts", import.meta.url), {
+    type: "module",
+  });
+
+  worker.addEventListener("message", incrementRenderCount);
+
+  worker.postMessage({ type: "init", canvas: offscreenCanvas }, [offscreenCanvas]);
+
+  onResize(canvas, ({ devicePixelSize }) => {
+    worker.postMessage({ type: "resize", size: devicePixelSize });
+  });
+</script>
+
+<Layout title="Offscreen gradient">
+  <canvas></canvas>
+</Layout>
+
+<style>
+  canvas {
+    aspect-ratio: 3 / 2;
+  }
+</style>

packages/gl/playground/src/pages/textures/texture.astro

@@ -5,26 +5,9 @@ import Layout from "../../layouts/Layout.astro";
 <script>
   import { glCanvas, loadTexture } from "@radiance/gl";
   import { incrementRenderCount } from "../../components/renderCount";
+  import type { TextureParams } from "@radiance/gl";
 
-  const texture = loadTexture("https://picsum.photos/id/669/600/400", {
-    placeholder: {
-      magFilter: "nearest",
-      data: new Uint8Array(
-        [
-          [255, 255, 255, 255],
-          [255, 255, 255, 255],
-          [255, 255, 255, 255],
-          [255, 255, 255, 255],
-          [99, 46, 22, 255],
-          [255, 255, 255, 255],
-        ].flat()
-      ),
-      width: 3,
-      height: 2,
-    },
-  });
-
-  const { onAfterRender } = glCanvas({
+  const { onAfterRender, uniforms } = glCanvas({
     canvas: "#glCanvas",
     fragment: /* glsl */ `
       in vec2 vUv;
@@ -57,10 +40,27 @@ import Layout from "../../layouts/Layout.astro";
       }
     `,
     uniforms: {
-      uTexture: texture,
+      uTexture: {
+        data: new Uint8Array(
+          [
+            [255, 255, 255, 255],
+            [255, 255, 255, 255],
+            [255, 255, 255, 255],
+            [255, 255, 255, 255],
+            [99, 46, 22, 255],
+            [255, 255, 255, 255],
+          ].flat(),
+        ),
+        width: 3,
+        height: 2,
+      } as TextureParams,
     },
   });
 
+  loadTexture("https://picsum.photos/id/669/600/400").then((texture) => {
+    uniforms.uTexture = texture;
+  });
+
   onAfterRender(incrementRenderCount);
 </script>
 

packages/gl/playground/src/workers/offscreenGradient.worker.ts

@@ -0,0 +1,77 @@
+import { glCanvas, loadTexture, type GLCanvas } from "@radiance/gl";
+
+type InitMessage = {
+  type: "init";
+  canvas: OffscreenCanvas;
+};
+
+type ResizeMessage = {
+  type: "resize";
+  size: {
+    width: number;
+    height: number;
+  };
+};
+
+type WorkerMessage = InitMessage | ResizeMessage;
+
+addEventListener("message", (event: MessageEvent<WorkerMessage>) => {
+  switch (event.data.type) {
+    case "init": {
+      init(event.data);
+      break;
+    }
+    case "resize": {
+      resize(event.data);
+      break;
+    }
+  }
+});
+
+let scene: GLCanvas;
+
+function init(message: InitMessage) {
+  scene = glCanvas({
+    canvas: message.canvas,
+    fragment: /* glsl */ `
+      in vec2 vUv;
+      uniform sampler2D uTexture;
+      uniform vec2 uResolution;
+      out vec4 fragColor;
+
+      #define CONTAIN 1
+      #define COVER 2
+      #define OBJECT_FIT CONTAIN
+
+      void main() {
+        vec2 textureResolution = vec2(textureSize(uTexture, 0));
+        float canvasRatio = uResolution.x / uResolution.y;
+        float textureRatio = textureResolution.x / textureResolution.y;
+
+        vec2 uv = vUv - 0.5;
+        if (OBJECT_FIT == CONTAIN ? canvasRatio > textureRatio : canvasRatio < textureRatio) {
+          uv.x *= canvasRatio / textureRatio;
+        } else {
+          uv.y *= textureRatio / canvasRatio;
+        }
+        uv += 0.5;
+
+        vec3 color = texture(uTexture, uv).rgb;
+        color *= step(0., uv.x) * (1. - step(1., uv.x));
+        color *= step(0., uv.y) * (1. - step(1., uv.y));
+
+        fragColor = vec4(color, 1.);
+      }
+    `,
+  });
+
+  scene.onAfterRender(() => {
+    postMessage({ type: "rendered" });
+  });
+
+  scene.uniforms.uTexture = loadTexture("https://picsum.photos/id/669/600/400");
+}
+
+function resize(message: ResizeMessage) {
+  scene?.setSize(message.size);
+}

packages/gl/playwright.config.ts

@@ -52,7 +52,9 @@ export default defineConfig({
     {
       name: "safari",
       use: { ...devices["Desktop Safari"], viewport: desktopViewport },
-      grepInvert: /video/, // the video does not work in the docker image used for the CI, maybe due to a codec issue
+      // - the video does not work in the docker image used for the CI, maybe due to a codec issue
+      // - issues with offscreen canvas and WebGL in Safari < 17
+      grepInvert: /video|offscreencanvas/,
     },
     {
       name: "android",
@@ -67,7 +69,9 @@ export default defineConfig({
     {
       name: "iphone",
       use: { ...devices["iPhone 12"], viewport: mobileViewport },
-      grepInvert: /video/, // the video does not work in the docker image used for the CI, maybe due to a codec issue
+      // - the video does not work in the docker image used for the CI, maybe due to a codec issue
+      // - issues with offscreen canvas and WebGL in Safari < 17
+      grepInvert: /video|offscreencanvas/,
     },
   ],
   webServer: {

packages/gl/src/core/texture.ts

@@ -1,3 +1,5 @@
+import { isHTMLImageTexture, isHTMLVideoTexture } from "../internal/typeGuards";
+
 const minFilterMap: Record<MinFilter, number> = {
   linear: WebGL2RenderingContext.LINEAR,
   nearest: WebGL2RenderingContext.NEAREST,
@@ -19,10 +21,9 @@ const wrapMap: Record<WrappingMode, number> = {
 /**
  * Given a WebGL2RenderingContext and a WebGLTexture, fill the texture with the given parameters.
  *
- * - if no src is provided, the data parameter will be used
- * - if a src is provided:
- *   - if it is an image that is loaded, or if it is a video that can start playing, they will be used to fill the texture
- *   - else, the placeholder parameter will be used, or a single black pixel will be used if no placeholder is specified
+ * - if a src is provided with a loaded media, it will be uploaded directly
+ * - if the src is an HTMLImageElement or HTMLVideoElement which is not loaded yet, a single black pixel is uploaded
+ * - if no src is provided, the data parameter will be used (with default to a single black pixel)
  *
  * @param gl - The WebGL2 context.
  * @param texture - The WebGL texture to fill.
@@ -53,15 +54,15 @@ export function fillTexture(
     magFilter = "linear",
     wrapS = "clamp-to-edge",
     wrapT = "clamp-to-edge",
-  } = (isLoadingMedia ? params.placeholder : params) || {};
+  } = params;
 
   gl.bindTexture(gl.TEXTURE_2D, texture);
   gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY);
 
   if (isLoadedMedia) {
     gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, format, type, params.src);
   } else {
-    const dataTexture = ((isLoadingMedia ? params.placeholder : params) || {}) as DataTextureParams;
+    const dataTexture = (params || {}) as DataTextureParams;
     const { data = new Uint8Array([0, 0, 0, 255]), width = 1, height = 1 } = dataTexture;
     gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, width, height, 0, format, type, data);
   }
@@ -89,52 +90,40 @@ export function fillTexture(
   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, wrapMap[wrapT]);
 }
 
-export function isHTMLImageTexture(
-  params: unknown,
-): params is ImageTextureParams & { src: HTMLImageElement } {
-  return (params as ImageTextureParams).src instanceof HTMLImageElement;
-}
-
-export function isHTMLVideoTexture(
-  params: unknown,
-): params is ImageTextureParams & { src: HTMLVideoElement } {
-  return (params as ImageTextureParams).src instanceof HTMLVideoElement;
-}
-
 /**
- * Loads an image from a URL and returns {@link ImageTextureParams}.
+ * Loads an image from a URL and returns a Promise resolving to {@link ImageTextureParams}.
  * @param src - URL of the image.
  * @param params - Additional texture parameters.
  */
-export function loadTexture<P extends Omit<ImageTextureParams, "src">>(src: string, params?: P) {
-  const img = document.createElement("img");
-
-  if (src.startsWith("http") && new URL(src).origin !== globalThis.location.origin) {
-    img.crossOrigin = "anonymous";
+export async function loadTexture<P extends BaseTextureParams>(
+  src: string,
+  params?: P,
+): Promise<ImageTextureParams<ImageBitmap>> {
+  const response = await fetch(src);
+  if (!response.ok) {
+    throw new Error(`Failed to load texture: ${src}`);
   }
+  const blob = await response.blob();
+  const bitmap = await createImageBitmap(blob, {
+    imageOrientation: params?.flipY === false ? "none" : "flipY",
+  });
 
-  const onload = () => {
-    img.removeEventListener("error", onerror);
-  };
-  const onerror = () => {
-    console.error(`Failed to load texture: ${src}`);
-    img.removeEventListener("load", onload);
-  };
-
-  img.addEventListener("load", onload, { once: true });
-  img.addEventListener("error", onerror, { once: true });
-
-  img.src = src;
-
-  return { ...(params as P), src: img };
+  return { ...params, src: bitmap };
 }
 
 /**
  * Loads a video from a URL and returns {@link ImageTextureParams}.
  * @param src - URL of the video.
  * @param params - Additional video loading options.
  */
-export function loadVideoTexture<P extends LoadVideoParams>(src: string, params?: P) {
+export function loadVideoTexture<P extends LoadVideoParams>(
+  src: string,
+  params?: P,
+): ImageTextureParams<HTMLVideoElement> {
+  if (typeof document === "undefined") {
+    throw new TypeError("loadVideoTexture requires a document context.");
+  }
+
   const video = document.createElement("video");
 
   video.loop = true;
@@ -176,7 +165,7 @@ export function loadVideoTexture<P extends LoadVideoParams>(src: string, params?
   video.src = src;
   video.load();
 
-  return { ...(params as P), src: video };
+  return { ...params, src: video };
 }
 
 /**
@@ -305,15 +294,11 @@ export type DataTextureParams = BaseTextureParams & {
 /**
  * Parameters for creating a texture from an external source (Image, Video, etc.).
  */
-export type ImageTextureParams = BaseTextureParams & {
+export type ImageTextureParams<S extends TexImageSource = TexImageSource> = BaseTextureParams & {
   /**
    * A source of texture: image, video, canvas, etc.
    */
-  src: TexImageSource;
-  /**
-   * Placeholder data to use while the source is loading.
-   */
-  placeholder?: DataTextureParams;
+  src: S;
 };
 
 /**
@@ -324,7 +309,7 @@ export type TextureParams = DataTextureParams | ImageTextureParams;
 /**
  * Parameters for loading a video texture.
  */
-export interface LoadVideoParams extends Omit<ImageTextureParams, "src"> {
+export interface LoadVideoParams extends BaseTextureParams {
   /**
    * Timecode in seconds from which to start the video.
    * @default 0