From 8572830c77abd283585cabb7d8813cc0212399c4 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 10:32:11 +0200 Subject: [PATCH 01/19] Add @fishjam-cloud/react-native-vision-camera-source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VisionCamera v5 frame outputs that publish camera frames to Fishjam — zero-copy forwarding and a WebGPU-rendered video source. - New package packages/react-native-vision-camera-source (useVisionCameraSource + /webgpu tier, forward/pooled track management, frame timestamp + orientation helpers). - react-client/mobile-client: useCustomSource gains a UseCustomSourceOptions argument so a source can publish its video as a regular camera track (videoType: 'camera' — a "virtual camera"). - webrtc-client: route createDataChannel through createNewConnection so a data-channel-first connection still wires ICE/track managers (tracks added later were silently never published). - Bump packages/react-native-webrtc to the custom-video-track native support. - Add the package to the release workflow + workspaces. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yaml | 23 + package.json | 3 +- .../plugin/src/withFishjamIos.ts | 2 +- packages/mobile-client/src/index.ts | 1 + packages/mobile-client/src/overrides/hooks.ts | 5 +- packages/mobile-client/src/overrides/types.ts | 8 + .../hooks/internal/useCustomSourceManager.ts | 14 +- .../react-client/src/hooks/useCustomSource.ts | 25 +- packages/react-client/src/index.ts | 2 +- packages/react-client/src/types/internal.ts | 13 + .../.eslintignore | 3 + .../.eslintrc | 13 + .../.prettierrc | 9 + .../README.md | 134 ++++ .../package.json | 92 +++ .../src/frameTimestamp.ts | 77 +++ .../src/index.ts | 23 + .../src/internal/useManagedCustomSource.ts | 30 + .../src/internal/useManagedForwardTrack.ts | 70 ++ .../src/internal/useManagedPooledTrack.ts | 104 +++ .../src/orientation.ts | 27 + .../src/useVisionCameraSource.ts | 137 ++++ .../src/webgpu.ts | 48 ++ .../src/webgpu/cameraPassthroughPipeline.ts | 152 ++++ .../src/webgpu/cameraShaderBindings.ts | 119 ++++ .../src/webgpu/cameraTextureResolver.ts | 69 ++ .../src/webgpu/cropUtilities.ts | 131 ++++ .../src/webgpu/frameRenderContext.ts | 61 ++ .../src/webgpu/requiredFeatures.ts | 53 ++ .../src/webgpu/useCameraWebGpuDevice.ts | 110 +++ .../src/webgpu/useVisionCameraWebGpuSource.ts | 325 +++++++++ .../src/webgpu/webGpuRuntime.ts | 21 + .../tsconfig.json | 18 + .../typedoc.json | 6 + packages/react-native-webrtc | 2 +- packages/webrtc-client/src/webRTCEndpoint.ts | 13 +- release-automation/bump-version.sh | 3 + yarn.lock | 648 +++++++++++++++++- 38 files changed, 2560 insertions(+), 34 deletions(-) create mode 100644 packages/react-native-vision-camera-source/.eslintignore create mode 100644 packages/react-native-vision-camera-source/.eslintrc create mode 100644 packages/react-native-vision-camera-source/.prettierrc create mode 100644 packages/react-native-vision-camera-source/README.md create mode 100644 packages/react-native-vision-camera-source/package.json create mode 100644 packages/react-native-vision-camera-source/src/frameTimestamp.ts create mode 100644 packages/react-native-vision-camera-source/src/index.ts create mode 100644 packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts create mode 100644 packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts create mode 100644 packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts create mode 100644 packages/react-native-vision-camera-source/src/orientation.ts create mode 100644 packages/react-native-vision-camera-source/src/useVisionCameraSource.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts create mode 100644 packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts create mode 100644 packages/react-native-vision-camera-source/tsconfig.json create mode 100644 packages/react-native-vision-camera-source/typedoc.json diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index edec7e53..30a935f5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -91,3 +91,26 @@ jobs: working-directory: packages/mobile-client - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }} working-directory: packages/mobile-client + + publish-react-native-vision-camera-source: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + - run: corepack enable + - run: yarn + - run: yarn workspace @fishjam-cloud/react-native-vision-camera-source run check:webrtc-published + - run: yarn build + - run: yarn pack --out package.tgz + working-directory: packages/react-native-vision-camera-source + - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }} + working-directory: packages/react-native-vision-camera-source diff --git a/package.json b/package.json index 71ac0a7f..8eb876af 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "packages/react-client", "packages/webrtc-client", "packages/mobile-client", + "packages/react-native-vision-camera-source", "packages/react-native-webrtc", "examples/react-client/*", "examples/mobile-client/*", @@ -36,7 +37,7 @@ }, "resolutions": { "jest-snapshot-prettier": "npm:prettier@^3", - "@types/react": "19.1.17" + "@types/react": "19.2.14" }, "devDependencies": { "@fishjam-cloud/protobufs": "workspace:^", diff --git a/packages/mobile-client/plugin/src/withFishjamIos.ts b/packages/mobile-client/plugin/src/withFishjamIos.ts index dab92eee..83f54626 100644 --- a/packages/mobile-client/plugin/src/withFishjamIos.ts +++ b/packages/mobile-client/plugin/src/withFishjamIos.ts @@ -14,7 +14,7 @@ function getSbeDisplayName(props: FishjamPluginOptions) { } const TARGETED_DEVICE_FAMILY = `"1,2"`; -const IPHONEOS_DEPLOYMENT_TARGET = '15.1'; +const IPHONEOS_DEPLOYMENT_TARGET = '17.0'; const GROUP_IDENTIFIER_TEMPLATE_REGEX = /{{GROUP_IDENTIFIER}}/gm; const BUNDLE_IDENTIFIER_TEMPLATE_REGEX = /{{BUNDLE_IDENTIFIER}}/gm; const DISPLAY_NAME_TEMPLATE_REGEX = /{{DISPLAY_NAME}}/gm; diff --git a/packages/mobile-client/src/index.ts b/packages/mobile-client/src/index.ts index 14654e76..56d8f0a9 100644 --- a/packages/mobile-client/src/index.ts +++ b/packages/mobile-client/src/index.ts @@ -71,6 +71,7 @@ export type { UseMicrophoneResult, UseScreenShareResult, UseCustomSourceResult, + UseCustomSourceOptions, UseInitializeDevicesReturn, Track, RemoteTrack, diff --git a/packages/mobile-client/src/overrides/hooks.ts b/packages/mobile-client/src/overrides/hooks.ts index 6ddf72b8..d822f888 100644 --- a/packages/mobile-client/src/overrides/hooks.ts +++ b/packages/mobile-client/src/overrides/hooks.ts @@ -3,6 +3,7 @@ import { type TracksMiddleware as ReactTracksMiddleware, useCamera as useCameraReactClient, useCustomSource as useCustomSourceReactClient, + type UseCustomSourceOptions, useInitializeDevices as useInitializeDevicesReactClient, useLivestreamStreamer as useLivestreamStreamerReactClient, useLivestreamViewer as useLivestreamViewerReactClient, @@ -83,8 +84,8 @@ export function useScreenShare(): UseScreenShareResult { }; } -export function useCustomSource(sourceId: T) { - const result = useCustomSourceReactClient(sourceId); +export function useCustomSource(sourceId: T, options?: UseCustomSourceOptions) { + const result = useCustomSourceReactClient(sourceId, options); return { ...result, stream: result.stream as RNMediaStream | undefined, diff --git a/packages/mobile-client/src/overrides/types.ts b/packages/mobile-client/src/overrides/types.ts index 4fcde84f..2c67b9a3 100644 --- a/packages/mobile-client/src/overrides/types.ts +++ b/packages/mobile-client/src/overrides/types.ts @@ -7,6 +7,7 @@ import type { Track as ReactClientTrack, useCamera as useCameraReactClient, useCustomSource as useCustomSourceReactClient, + UseCustomSourceOptions as ReactClientUseCustomSourceOptions, useInitializeDevices as useInitializeDevicesReactClient, UseLivestreamStreamerResult as ReactClientUseLivestreamStreamerResult, UseLivestreamViewerResult as ReactClientUseLivestreamViewerResult, @@ -102,6 +103,13 @@ export type UseCustomSourceResult = Omit Promise; }; +/** + * Options controlling how a custom source's tracks are published. + * Pass `videoType: 'camera'` to publish the source's video as a regular camera + * track (a "virtual camera"), so receivers bucket it as the peer's camera track. + */ +export type UseCustomSourceOptions = ReactClientUseCustomSourceOptions; + export type UseInitializeDevicesReturn = { initializeDevices: ( ...args: Parameters['initializeDevices']> diff --git a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts index d5456f1b..7469daa4 100644 --- a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts +++ b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts @@ -1,7 +1,7 @@ import { type FishjamClient, type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client"; import { useCallback, useEffect, useMemo, useState } from "react"; -import type { CustomSourceState, CustomSourceTracks } from "../../types/internal"; +import type { CustomSourceState, CustomSourceTracks, CustomSourceTrackTypes } from "../../types/internal"; import type { PeerStatus } from "../../types/public"; type CustomSourceManagerProps = { @@ -11,7 +11,7 @@ type CustomSourceManagerProps = { }; export type CustomSourceManager = { - setStream: (sourceId: string, stream: MediaStream | null) => Promise; + setStream: (sourceId: string, stream: MediaStream | null, trackTypes?: CustomSourceTrackTypes) => Promise; getSource: (sourceId: string) => CustomSourceState | undefined; }; @@ -55,12 +55,14 @@ export function useCustomSourceManager({ const promises = []; const displayName = getDisplayName(); + const videoType = source.trackTypes?.videoType ?? "customVideo"; + const audioType = source.trackTypes?.audioType ?? "customAudio"; if (video) { - const videoMetadata = { type: "customVideo", displayName, paused: false } as const; + const videoMetadata = { type: videoType, displayName, paused: false } as const; promises.push(addTrackToFishjamClient(video, videoMetadata)); } if (audio) { - const audioMetadata = { type: "customAudio", displayName, paused: false } as const; + const audioMetadata = { type: audioType, displayName, paused: false } as const; promises.push(addTrackToFishjamClient(audio, audioMetadata)); } @@ -87,14 +89,14 @@ export function useCustomSourceManager({ const getSource = useCallback((sourceId: string) => sources[sourceId], [sources]); const setStream = useCallback( - async (sourceId: string, stream: MediaStream | null) => { + async (sourceId: string, stream: MediaStream | null, trackTypes?: CustomSourceTrackTypes) => { const oldSource = sources[sourceId]; if (stream === oldSource?.stream) return; if (oldSource?.trackIds) await removeTracks(oldSource.trackIds); if (stream !== null) { - setSources((old) => ({ ...old, [sourceId]: { stream } })); + setSources((old) => ({ ...old, [sourceId]: { stream, trackTypes } })); return; } if (!oldSource) return; diff --git a/packages/react-client/src/hooks/useCustomSource.ts b/packages/react-client/src/hooks/useCustomSource.ts index 8ceaf9d8..befdebdf 100644 --- a/packages/react-client/src/hooks/useCustomSource.ts +++ b/packages/react-client/src/hooks/useCustomSource.ts @@ -1,12 +1,28 @@ import { useCallback, useContext, useMemo } from "react"; import { CustomSourceContext } from "../contexts/customSource"; +import type { CustomSourceTrackTypes } from "../types/internal"; + +/** + * Options controlling how a custom source's tracks are published. + * @group Hooks + */ +export type UseCustomSourceOptions = CustomSourceTrackTypes; /** * This hook can register/deregister a custom MediaStream with Fishjam. + * + * By default a custom source's video is published as a `customVideo` track and + * its audio as a `customAudio` track. Pass `options.videoType: 'camera'` to + * publish the video as a regular camera track (a "virtual camera"), so every + * receiver buckets it as the peer's camera track with no receiver-side changes. + * + * @param sourceId - Stable id identifying this custom source. + * @param options - Optional track types to publish under. Defaults to + * `{ videoType: 'customVideo', audioType: 'customAudio' }`. * @group Hooks */ -export function useCustomSource(sourceId: T) { +export function useCustomSource(sourceId: T, options?: UseCustomSourceOptions) { const customSourceManager = useContext(CustomSourceContext); if (!customSourceManager) throw Error("useCustomSource must be used within FishjamProvider"); @@ -14,9 +30,12 @@ export function useCustomSource(sourceId: T) { const stream = useMemo(() => getSource(sourceId)?.stream, [getSource, sourceId]); + const videoType = options?.videoType; + const audioType = options?.audioType; + const setStream = useCallback( - (newStream: MediaStream | null) => managerSetStream(sourceId, newStream), - [managerSetStream, sourceId], + (newStream: MediaStream | null) => managerSetStream(sourceId, newStream, { videoType, audioType }), + [managerSetStream, sourceId, videoType, audioType], ); return { diff --git a/packages/react-client/src/index.ts b/packages/react-client/src/index.ts index 5f404ec4..d1fd6510 100644 --- a/packages/react-client/src/index.ts +++ b/packages/react-client/src/index.ts @@ -9,7 +9,7 @@ export { useInitializeDevices, UseInitializeDevicesParams } from "./hooks/device export { useMicrophone } from "./hooks/devices/useMicrophone"; export { InitializeDevicesSettings } from "./hooks/internal/devices/useMediaDevices"; export { type JoinRoomConfig, useConnection } from "./hooks/useConnection"; -export { useCustomSource } from "./hooks/useCustomSource"; +export { useCustomSource, type UseCustomSourceOptions } from "./hooks/useCustomSource"; export { useDataChannel } from "./hooks/useDataChannel"; export { type ConnectStreamerConfig, diff --git a/packages/react-client/src/types/internal.ts b/packages/react-client/src/types/internal.ts index f7f3abdb..31f003eb 100644 --- a/packages/react-client/src/types/internal.ts +++ b/packages/react-client/src/types/internal.ts @@ -47,7 +47,20 @@ export type CustomSourceTracks = { audioId?: string; }; +/** + * Track types a custom source may publish under. Defaults to the "custom" + * variants, but a source can opt into publishing its video as a regular + * `camera` track (a virtual camera) so receivers bucket it as a camera track. + */ +export type CustomSourceTrackTypes = { + /** Metadata type for the source's video track. Defaults to `customVideo`. */ + videoType?: "camera" | "customVideo"; + /** Metadata type for the source's audio track. Defaults to `customAudio`. */ + audioType?: "customAudio"; +}; + export type CustomSourceState = { stream: MediaStream; trackIds?: CustomSourceTracks; + trackTypes?: CustomSourceTrackTypes; }; diff --git a/packages/react-native-vision-camera-source/.eslintignore b/packages/react-native-vision-camera-source/.eslintignore new file mode 100644 index 00000000..28bd0deb --- /dev/null +++ b/packages/react-native-vision-camera-source/.eslintignore @@ -0,0 +1,3 @@ +dist +build +coverage diff --git a/packages/react-native-vision-camera-source/.eslintrc b/packages/react-native-vision-camera-source/.eslintrc new file mode 100644 index 00000000..a97fd5e1 --- /dev/null +++ b/packages/react-native-vision-camera-source/.eslintrc @@ -0,0 +1,13 @@ +{ + "extends": ["../../.eslintrc.js", "expo", "prettier"], + "plugins": ["prettier"], + "ignorePatterns": ["lib", "/dist/*"], + "rules": { + "prettier/prettier": "error" + }, + "settings": { + "import/resolver": { + "typescript": true + } + } +} diff --git a/packages/react-native-vision-camera-source/.prettierrc b/packages/react-native-vision-camera-source/.prettierrc new file mode 100644 index 00000000..3ce79d75 --- /dev/null +++ b/packages/react-native-vision-camera-source/.prettierrc @@ -0,0 +1,9 @@ +{ + "bracketSameLine": true, + "printWidth": 120, + "quoteProps": "consistent", + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "all", + "useTabs": false +} diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md new file mode 100644 index 00000000..4e21fe7f --- /dev/null +++ b/packages/react-native-vision-camera-source/README.md @@ -0,0 +1,134 @@ +# `@fishjam-cloud/react-native-vision-camera-source` + +Publish a [VisionCamera](https://react-native-vision-camera.com) (v5) feed to +[Fishjam](https://fishjam.io) — as-is, with inference worklets running on the same frames, or with +your own WebGPU rendering drawn into the published video. + +The hooks follow the Fishjam source-hook family (`useCamera`, `useScreenShare`, +`useCustomSource`): they create the underlying track, publish it, and clean up on unmount. Your +component stays fully declarative. + +## Prerequisites + +- `react-native-vision-camera` **v5** and `react-native-vision-camera-worklets` +- [`react-native-worklets`](https://docs.swmansion.com/react-native-worklets/) with its Babel + plugin configured (required by VisionCamera's frame outputs) +- `@fishjam-cloud/react-native-client` with your app wrapped in `FishjamProvider` +- New Architecture (custom video tracks require it) +- For the `/webgpu` entry: `react-native-webgpu` ≥ 0.5.15 (optional otherwise) + +## Publish the camera + +```tsx +import { useCamera as useVisionCamera, useCameraDevices, useCameraPermission } from 'react-native-vision-camera'; +import { RTCView } from '@fishjam-cloud/react-native-client'; +import { useVisionCameraSource } from '@fishjam-cloud/react-native-vision-camera-source'; + +function CameraPublisher() { + const { hasPermission } = useCameraPermission(); + const cameraDevice = useCameraDevices().find((device) => device.position === 'front'); + + const { frameOutput, stream } = useVisionCameraSource('my-camera', { videoType: 'camera' }); + + useVisionCamera({ device: cameraDevice, isActive: hasPermission, outputs: [frameOutput] }); + + return stream ? : null; // self-view +} +``` + +Frames are handed to Fishjam without copying pixels. `videoType: 'camera'` makes receivers treat +the video as the peer's camera track; omit it to publish a separate custom video track. + +## Publish + run inference + +```tsx +const onFrame = useCallback( + (frame: Frame) => { + 'worklet'; + const pose = detectPose(frame); // any VisionCamera frame-processor plugin + poseResults.setBlocking(pose); + }, + [detectPose], +); + +const { frameOutput } = useVisionCameraSource('my-camera', { videoType: 'camera', onFrame }); +``` + +The frame is valid only inside your synchronous callback — the hook releases it afterwards. + +## Render with WebGPU + +The `/webgpu` entry draws your own content — shaders, overlays, effects — into the published +video with zero pixel copies. The hook owns the output surfaces, GPU synchronization with the +encoder, timestamps, and frame lifetimes; your worklet only encodes passes. + +```tsx +import { + useVisionCameraWebGpuSource, + useCameraWebGpuDevice, + createCameraShaderBindings, + getOutputSurfaceFormat, + type WebGpuFrameRenderFunction, +} from '@fishjam-cloud/react-native-vision-camera-source/webgpu'; + +const { device } = useCameraWebGpuDevice(); +const effect = useMemo(() => { + if (device == null) return null; + const cameraBindings = createCameraShaderBindings(device); + const pipeline = device.createRenderPipeline({ + layout: device.createPipelineLayout({ bindGroupLayouts: [cameraBindings.bindGroupLayout] }), + vertex: { module: device.createShaderModule({ code: FULL_SCREEN_TRIANGLE_WGSL }) }, + fragment: { + module: device.createShaderModule({ + code: + cameraBindings.shaderCode + + `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f { + let color = sampleCamera(uv); // upright RGB on BOTH platforms + return vec4f(vec3f(dot(color.rgb, vec3f(0.299, 0.587, 0.114))), 1.0); // grayscale + }`, + }), + targets: [{ format: getOutputSurfaceFormat() }], + }, + }); + return { cameraBindings, pipeline }; +}, [device]); + +const onFrame = useCallback( + (frame: Frame, render: WebGpuFrameRenderFunction) => { + 'worklet'; + if (effect == null) return; // drop until the pipeline is ready + render(({ commandEncoder, outputTexture, cameraBindGroup }) => { + const pass = commandEncoder.beginRenderPass({ + colorAttachments: [{ view: outputTexture.createView(), loadOp: 'clear', storeOp: 'store' }], + }); + pass.setPipeline(effect.pipeline); + pass.setBindGroup(0, cameraBindGroup!); + pass.draw(3); + pass.end(); + }); + }, + [effect], +); + +const { frameOutput, stream } = useVisionCameraWebGpuSource('my-camera', { + videoType: 'camera', + width: 720, + height: 1280, + cameraShaderBindings: effect?.cameraBindings, + onFrame, +}); +useVisionCamera({ device: cameraDevice, isActive: true, outputs: [frameOutput] }); +``` + +Prefer zero WGSL? `createCameraPassthroughPipeline` + `encodeCameraPassthrough` publish the +camera through the same pipeline (crop and platform color handling included) and compose with +your own overlay passes. Pipelines that cannot sample `texture_external` can resolve the camera +into a plain texture with `createCameraTextureResolver`. + +> Verify camera + WebGPU behavior on **physical devices** — the iOS Simulator cannot import the +> camera's YUV textures. + +## Development + +This package is part of the [web-client-sdk](https://github.com/fishjam-cloud/web-client-sdk) +monorepo. `yarn build` compiles `src/` to `dist/` with `tsc`. diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json new file mode 100644 index 00000000..88f0d29a --- /dev/null +++ b/packages/react-native-vision-camera-source/package.json @@ -0,0 +1,92 @@ +{ + "name": "@fishjam-cloud/react-native-vision-camera-source", + "version": "0.28.3", + "description": "VisionCamera frame outputs that publish camera frames to Fishjam — zero-copy forwarding and WebGPU-rendered video sources", + "license": "Apache-2.0", + "author": "Fishjam Team", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "react-native": "dist/index.js", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "react-native": "./dist/index.js", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./webgpu": { + "types": "./dist/webgpu.d.ts", + "react-native": "./dist/webgpu.js", + "import": "./dist/webgpu.js", + "default": "./dist/webgpu.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/**" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/fishjam-cloud/web-client-sdk.git", + "directory": "packages/react-native-vision-camera-source" + }, + "homepage": "https://github.com/fishjam-cloud/web-client-sdk#readme", + "bugs": "https://github.com/fishjam-cloud/web-client-sdk/issues", + "keywords": [ + "webrtc", + "fishjam", + "vision-camera", + "webgpu" + ], + "scripts": { + "tsc": "tsc", + "build:webrtc-workspace": "yarn workspace @fishjam-cloud/react-native-webrtc run prepare", + "build": "yarn build:webrtc-workspace && tsc", + "lint": "eslint . --ext .ts,.tsx --fix", + "prepare": "yarn build:webrtc-workspace && tsc", + "format": "prettier --write . --ignore-path ./.eslintignore", + "check:webrtc-published": "node ../../release-automation/check-webrtc-published.mjs" + }, + "lint-staged": { + "*": [ + "yarn format" + ], + "*.{js,ts,tsx}": [ + "yarn lint" + ] + }, + "peerDependencies": { + "@fishjam-cloud/react-native-client": "workspace:*", + "@fishjam-cloud/react-native-webrtc": "workspace:*", + "react": "*", + "react-native": "*", + "react-native-vision-camera": "^5.0.0", + "react-native-vision-camera-worklets": "^5.0.0", + "react-native-webgpu": ">=0.5.15" + }, + "peerDependenciesMeta": { + "react-native-webgpu": { + "optional": true + } + }, + "dependencies": { + "react-native-worklets": ">=0.9.0" + }, + "devDependencies": { + "@fishjam-cloud/react-native-client": "workspace:*", + "@fishjam-cloud/react-native-webrtc": "workspace:*", + "@types/react": "19.2.14", + "@webgpu/types": "0.1.65", + "eslint-config-expo": "~9.2.0", + "eslint-import-resolver-typescript": "^3.10.1", + "eslint-plugin-prettier": "^5.5.1", + "react": "19.2.3", + "react-native": "0.85.3", + "react-native-nitro-modules": "0.35.7", + "react-native-vision-camera": "5.0.10", + "react-native-vision-camera-worklets": "5.0.10", + "react-native-webgpu": "0.5.15", + "typescript": "^5.8.3" + }, + "packageManager": "yarn@4.16.0" +} diff --git a/packages/react-native-vision-camera-source/src/frameTimestamp.ts b/packages/react-native-vision-camera-source/src/frameTimestamp.ts new file mode 100644 index 00000000..db59cf26 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/frameTimestamp.ts @@ -0,0 +1,77 @@ +import { createSynchronizable, type Synchronizable } from 'react-native-worklets'; + +// VisionCamera 5.x reports Frame.timestamp in platform-inconsistent units: seconds on iOS +// (CMTime.seconds) and nanoseconds on Android (the raw CameraX sensor timestamp). Rather than +// branching on the platform — which would silently break if VisionCamera ever aligns the units — +// we discriminate by magnitude: seconds-since-boot values sit around 1e5–1e6 while +// nanoseconds-since-boot values sit around 1e14, so any value at or above this threshold is +// already in nanoseconds. +const NANOSECONDS_MAGNITUDE_THRESHOLD = 1e12; + +const NANOSECONDS_PER_SECOND = 1e9; + +interface FrameTimestampTimeline { + /** Absolute timestamp of the first frame, in nanoseconds; the timeline's zero point. */ + firstNanoseconds: number | null; + /** Last emitted timeline value, in nanoseconds; used to reject non-monotonic timestamps. */ + lastNanoseconds: number; +} + +/** + * Per-track timestamp timeline. A `Synchronizable` box (not a plain object) so the state survives + * frame-callback re-registration: a plain captured object is copied into each new worklet closure, + * which would restart the timeline mid-stream and emit timestamps that jump backwards. + */ +export type FrameTimestampState = Synchronizable; + +export function createFrameTimestampState(): FrameTimestampState { + // lastNanoseconds starts below zero so the first frame (offset 0) passes the monotonicity check. + return createSynchronizable({ firstNanoseconds: null, lastNanoseconds: -1 }); +} + +/** + * Converts a raw VisionCamera `Frame.timestamp` into a monotonic nanosecond timeline that starts + * at zero on the first frame, preserving the real gaps between frames for encoder pacing. + * + * Returns `null` when the frame carries no usable timestamp (invalid frames report `0`) or when + * the value does not advance the timeline — callers should then fall back to their tier's default + * (omit the timestamp so the native layer stamps a monotonic clock, or synthesize from the frame + * interval). + */ +export function normalizeFrameTimestampNanoseconds(state: FrameTimestampState, rawTimestamp: number): number | null { + 'worklet'; + if (!(rawTimestamp > 0)) { + return null; + } + const absoluteNanoseconds = + rawTimestamp >= NANOSECONDS_MAGNITUDE_THRESHOLD ? rawTimestamp : rawTimestamp * NANOSECONDS_PER_SECOND; + const timeline = state.getBlocking(); + const firstNanoseconds = timeline.firstNanoseconds ?? absoluteNanoseconds; + const timelineNanoseconds = absoluteNanoseconds - firstNanoseconds; + if (timelineNanoseconds <= timeline.lastNanoseconds) { + return null; + } + state.setBlocking({ firstNanoseconds, lastNanoseconds: timelineNanoseconds }); + return timelineNanoseconds; +} + +/** + * Like {@link normalizeFrameTimestampNanoseconds}, but never fails: when the frame carries no + * usable timestamp, it advances the timeline by `frameIntervalNanoseconds` instead. For sinks + * that require a timestamp on every frame (`pushFrame`). + */ +export function nextFrameTimestampNanoseconds( + state: FrameTimestampState, + rawTimestamp: number, + frameIntervalNanoseconds: number, +): number { + 'worklet'; + const normalized = normalizeFrameTimestampNanoseconds(state, rawTimestamp); + if (normalized != null) { + return normalized; + } + const timeline = state.getBlocking(); + const fallbackNanoseconds = timeline.lastNanoseconds < 0 ? 0 : timeline.lastNanoseconds + frameIntervalNanoseconds; + state.setBlocking({ firstNanoseconds: timeline.firstNanoseconds, lastNanoseconds: fallbackNanoseconds }); + return fallbackNanoseconds; +} diff --git a/packages/react-native-vision-camera-source/src/index.ts b/packages/react-native-vision-camera-source/src/index.ts new file mode 100644 index 00000000..5f356255 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/index.ts @@ -0,0 +1,23 @@ +/** + * VisionCamera → Fishjam adapter. + * + * Source hooks that publish a VisionCamera feed to Fishjam, in the same family as + * `useCustomSource`: the hook owns the track, the publishing, and the cleanup, and hands you a + * VisionCamera frame output to plug into your own camera session. + * + * - {@link useVisionCameraSource} — publish the camera feed as-is (copy-free), optionally running + * inference worklets on the same frames. + * - `useVisionCameraWebGpuSource` (from `@fishjam-cloud/react-native-vision-camera-source/webgpu`) + * — render your own WebGPU content into the published video. + * + * Requires VisionCamera 5 and `FishjamProvider` from `@fishjam-cloud/react-native-client`. + * + * @packageDocumentation + */ + +export { rotationDegreesFromOrientation } from './orientation'; +export { + useVisionCameraSource, + type UseVisionCameraSourceOptions, + type UseVisionCameraSourceResult, +} from './useVisionCameraSource'; diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts b/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts new file mode 100644 index 00000000..ee0f5388 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts @@ -0,0 +1,30 @@ +import { useCustomSource } from '@fishjam-cloud/react-native-client'; +import type { MediaStream } from '@fishjam-cloud/react-native-webrtc'; +import { useEffect, useRef } from 'react'; + +/** + * Publishes `stream` under `sourceId` via the SDK's custom-source machinery: publish when the + * stream appears, unpublish on cleanup. Must be used under `FishjamProvider`. + */ +export function useManagedCustomSource( + sourceId: string, + videoType: 'camera' | 'customVideo' | undefined, + stream: MediaStream | null, +): void { + const { setStream } = useCustomSource(sourceId, videoType != null ? { videoType } : undefined); + + // setStream is not referentially stable across renders; going through a ref keeps the effect's + // cleanup from unpublishing with a stale instance. + const setStreamRef = useRef(setStream); + setStreamRef.current = setStream; + + useEffect(() => { + if (stream == null) { + return; + } + void setStreamRef.current(stream); + return () => { + void setStreamRef.current(null); + }; + }, [stream]); +} diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts b/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts new file mode 100644 index 00000000..72cde124 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts @@ -0,0 +1,70 @@ +import { + createCustomVideoTrack, + type CustomVideoTrackResult, + type ForwardTrack, + type MediaStream, +} from '@fishjam-cloud/react-native-webrtc'; +import { useEffect, useState } from 'react'; + +interface ManagedForwardTrack { + track: ForwardTrack | null; + stream: MediaStream | null; + error: Error | null; +} + +const INITIAL_STATE: ManagedForwardTrack = { track: null, stream: null, error: null }; + +function stopStreamTracks(stream: MediaStream): void { + try { + stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); + } catch (cause) { + console.warn('useManagedForwardTrack: stopping tracks failed', cause); + } +} + +/** + * Owns the async lifecycle of a forwarding custom video track: creates it while `enabled`, + * exposes the handle + stream once ready, and stops the tracks on disable/unmount (also when + * creation resolves after the owner already unmounted). + */ +export function useManagedForwardTrack(enabled: boolean): ManagedForwardTrack { + const [managedTrack, setManagedTrack] = useState(INITIAL_STATE); + + useEffect(() => { + // The disabled case needs no work: the previous run's cleanup already reset the state. + if (!enabled) { + return; + } + let cancelled = false; + let created: CustomVideoTrackResult | null = null; + + createCustomVideoTrack() + .then((result) => { + created = result; + if (cancelled) { + stopStreamTracks(result.stream); + return; + } + setManagedTrack({ track: result.track, stream: result.stream, error: null }); + }) + .catch((cause: unknown) => { + if (!cancelled) { + setManagedTrack({ + track: null, + stream: null, + error: cause instanceof Error ? cause : new Error(String(cause)), + }); + } + }); + + return () => { + cancelled = true; + if (created != null) { + stopStreamTracks(created.stream); + } + setManagedTrack(INITIAL_STATE); + }; + }, [enabled]); + + return managedTrack; +} diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts new file mode 100644 index 00000000..ba0ea3ab --- /dev/null +++ b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts @@ -0,0 +1,104 @@ +import { + createCustomVideoBufferPool, + createCustomVideoTrack, + type CustomVideoBufferPool, + type CustomVideoTrackResult, + type MediaStream, + type PooledTrack, +} from '@fishjam-cloud/react-native-webrtc'; +import { useEffect, useState } from 'react'; + +/** One pooled output surface as plain values the frame worklet can capture and import itself. */ +export interface WorkletBufferDescriptor { + index: number; + surfaceHandle: bigint; + width: number; + height: number; +} + +interface ManagedPooledTrack { + track: PooledTrack | null; + stream: MediaStream | null; + /** Plain per-surface descriptors (the pool object itself is not worklet-serializable). */ + bufferDescriptors: WorkletBufferDescriptor[] | null; + error: Error | null; +} + +const INITIAL_STATE: ManagedPooledTrack = { track: null, stream: null, bufferDescriptors: null, error: null }; + +function disposeCreated(pool: CustomVideoBufferPool | null, created: CustomVideoTrackResult | null): void { + // Stop the track first, then free the pool — the pool must outlive the track's last frame. + try { + created?.stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); + } catch (cause) { + console.warn('useManagedPooledTrack: stopping tracks failed', cause); + } + void pool?.dispose().catch((cause: unknown) => { + console.warn('useManagedPooledTrack: disposing the buffer pool failed', cause); + }); +} + +/** + * Owns the async lifecycle of a surface pool + pooled custom video track: allocates both while + * `enabled` (re-allocates when the dimensions change), exposes worklet-ready descriptors, and + * tears down in the correct order (stop tracks, then dispose the pool) on disable/unmount. + */ +export function useManagedPooledTrack( + enabled: boolean, + width: number, + height: number, + poolSize: number, +): ManagedPooledTrack { + const [managedTrack, setManagedTrack] = useState(INITIAL_STATE); + + useEffect(() => { + // The disabled case needs no work: the previous run's cleanup already reset the state (so + // consumers never hold descriptors of a disposed pool). + if (!enabled) { + return; + } + let cancelled = false; + let pool: CustomVideoBufferPool | null = null; + let created: CustomVideoTrackResult | null = null; + + (async () => { + pool = await createCustomVideoBufferPool({ width, height, poolSize }); + if (cancelled) { + disposeCreated(pool, null); + return; + } + created = await createCustomVideoTrack({ pool }); + if (cancelled) { + disposeCreated(pool, created); + return; + } + const bufferDescriptors = pool.buffers.map((buffer) => ({ + index: buffer.index, + surfaceHandle: buffer.surfaceHandle, + width: buffer.width, + height: buffer.height, + })); + setManagedTrack({ track: created.track, stream: created.stream, bufferDescriptors, error: null }); + })().catch((cause: unknown) => { + if (!cancelled) { + setManagedTrack({ + track: null, + stream: null, + bufferDescriptors: null, + error: cause instanceof Error ? cause : new Error(String(cause)), + }); + } + disposeCreated(pool, created); + pool = null; + created = null; + }); + + return () => { + cancelled = true; + disposeCreated(pool, created); + setManagedTrack(INITIAL_STATE); + }; + }, [enabled, width, height, poolSize]); + + return managedTrack; +} diff --git a/packages/react-native-vision-camera-source/src/orientation.ts b/packages/react-native-vision-camera-source/src/orientation.ts new file mode 100644 index 00000000..8b1512fe --- /dev/null +++ b/packages/react-native-vision-camera-source/src/orientation.ts @@ -0,0 +1,27 @@ +import type { CameraOrientation } from 'react-native-vision-camera'; + +/** + * Maps a VisionCamera frame orientation to the clockwise rotation, in degrees, needed to bring the + * frame's pixel data upright: `'up'` → 0, `'right'` → 90, `'down'` → 180, `'left'` → 270. + * + * The source hooks apply this automatically. It is exported for advanced consumers who wire the + * low-level frame APIs from `@fishjam-cloud/react-native-webrtc` themselves. + * + * Worklet-safe: call it from a frame callback or from the JS thread. + * + * @param orientation A VisionCamera `Frame.orientation` value. + * @group Utilities + */ +export function rotationDegreesFromOrientation(orientation: CameraOrientation): 0 | 90 | 180 | 270 { + 'worklet'; + if (orientation === 'right') { + return 90; + } + if (orientation === 'down') { + return 180; + } + if (orientation === 'left') { + return 270; + } + return 0; +} diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts new file mode 100644 index 00000000..c2796cac --- /dev/null +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -0,0 +1,137 @@ +import { forwardFrame, type MediaStream } from '@fishjam-cloud/react-native-webrtc'; +import { useMemo } from 'react'; +import { + type CameraFrameOutput, + type Frame, + type FrameDroppedReason, + type FrameOutputOptions, + useFrameOutput, +} from 'react-native-vision-camera'; + +import { createFrameTimestampState, normalizeFrameTimestampNanoseconds } from './frameTimestamp'; +import { useManagedCustomSource } from './internal/useManagedCustomSource'; +import { useManagedForwardTrack } from './internal/useManagedForwardTrack'; +import { rotationDegreesFromOrientation } from './orientation'; + +/** + * Options for {@link useVisionCameraSource}. Also accepts every VisionCamera frame-output option + * (`targetResolution`, `pixelFormat`, `dropFramesWhileBusy`, and so on) and passes it through. + */ +export interface UseVisionCameraSourceOptions extends Partial { + /** + * Whether the source is live. While `false`, no track exists and nothing is published — the + * declarative sibling of VisionCamera's `isActive`. Defaults to `true`. + */ + enabled?: boolean; + /** + * How receivers see the published video — same semantics as `useCustomSource`: + * `'camera'` publishes it as the peer's camera track (a "virtual camera"), `'customVideo'` + * (the default) as a separate custom video track. + */ + videoType?: 'camera' | 'customVideo'; + /** + * Optional worklet called with every camera frame, after the frame has been sent to Fishjam — + * run your frame-processor plugins (pose detection, OCR, …) here. + * + * The frame is valid only for the duration of this synchronous callback; the hook releases it + * when the callback returns, so do not retain it. Keep the function's identity stable + * (`useCallback` or module scope) — a new function every render forces VisionCamera to + * re-register the frame callback. + */ + onFrame?: (frame: Frame) => void; + /** Called whenever the camera pipeline drops a frame; forwarded to VisionCamera. */ + onFrameDropped?: (reason: FrameDroppedReason) => void; +} + +/** Result of {@link useVisionCameraSource}. */ +export interface UseVisionCameraSourceResult { + /** + * The VisionCamera frame output driving this source. Plug it into your camera session: + * `useCamera({ device, isActive, outputs: [frameOutput] })`. + */ + frameOutput: CameraFrameOutput; + /** + * The published stream — render it with `RTCView` for a self-view. `null` until the + * underlying track is ready (creation is asynchronous). + */ + stream: MediaStream | null; + /** Failure while creating the underlying track, if any. */ + error: Error | null; +} + +/** + * Publishes your VisionCamera feed to Fishjam. + * + * A sibling of `useCustomSource`: the hook creates the video track, publishes it under + * `sourceId`, and cleans everything up on unmount. Each camera frame is handed to Fishjam + * without copying its pixels. Must be used under `FishjamProvider`. + * + * ```tsx + * const { frameOutput, stream } = useVisionCameraSource('my-camera', { videoType: 'camera' }); + * + * useVisionCamera({ device: cameraDevice, isActive: true, outputs: [frameOutput] }); + * + * return stream ? : null; + * ``` + * + * To also run inference on the same frames, pass an {@link UseVisionCameraSourceOptions.onFrame | onFrame} + * worklet. To render your own content into the published video with WebGPU, use + * `useVisionCameraWebGpuSource` from `@fishjam-cloud/react-native-vision-camera-source/webgpu` + * instead. + * + * @param sourceId Identifies this source among the peer's tracks, like in `useCustomSource`. + * @param options See {@link UseVisionCameraSourceOptions}. + * @group Hooks + */ +export function useVisionCameraSource( + sourceId: SourceId, + options: UseVisionCameraSourceOptions = {}, +): UseVisionCameraSourceResult { + const { enabled = true, videoType, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; + + const { track, stream, error } = useManagedForwardTrack(enabled); + useManagedCustomSource(sourceId, videoType, stream); + + const timestampState = useMemo(() => createFrameTimestampState(), []); + + const handleFrame = useMemo(() => { + return (frame: Frame) => { + 'worklet'; + try { + if (track != null) { + const nativeBuffer = frame.getNativeBuffer(); + try { + const timestampNanoseconds = normalizeFrameTimestampNanoseconds(timestampState, frame.timestamp); + forwardFrame(track, { + nativeBuffer: nativeBuffer.pointer, + rotation: rotationDegreesFromOrientation(frame.orientation), + // When the frame carries no usable timestamp, omit it — the native layer then + // stamps the frame with its own monotonic clock. + ...(timestampNanoseconds != null ? { timestampNs: timestampNanoseconds } : {}), + }); + } finally { + nativeBuffer.release(); + } + } + if (userOnFrame != null) { + userOnFrame(frame); + } + } catch (cause) { + console.warn('useVisionCameraSource: processing a camera frame failed: ' + String(cause)); + } finally { + frame.dispose(); + } + }; + }, [track, userOnFrame, timestampState]); + + const frameOutput = useFrameOutput({ + // 'native' keeps the pipeline copy-free; override via options if a plugin needs 'rgb'/'yuv'. + pixelFormat: 'native', + dropFramesWhileBusy: true, + ...frameOutputOptions, + onFrame: handleFrame, + onFrameDropped, + }); + + return { frameOutput, stream, error }; +} diff --git a/packages/react-native-vision-camera-source/src/webgpu.ts b/packages/react-native-vision-camera-source/src/webgpu.ts new file mode 100644 index 00000000..f22984fa --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu.ts @@ -0,0 +1,48 @@ +/// +/** + * WebGPU tier of the VisionCamera → Fishjam adapter — render your own content into the published + * video. Requires `react-native-webgpu` (an optional peer of this package; only this entry point + * loads it). + * + * - {@link useVisionCameraWebGpuSource} — the source hook: camera in, your WebGPU passes, + * published video out. + * - {@link createCameraShaderBindings} — sample the live camera from your own shaders via + * `sampleCamera(uv)`, with the platform's YUV decode handled for you. + * - {@link createCameraPassthroughPipeline} / {@link encodeCameraPassthrough} — a ready-made + * camera→output pass to publish the camera with zero WGSL, or to build overlays on. + * - {@link createCameraTextureResolver} — opt-in plain-texture camera for pipelines that can't + * sample `texture_external`. + * + * @packageDocumentation + */ + +export { + type CameraPassthroughPipeline, + type CameraPassthroughPipelineOptions, + createCameraPassthroughPipeline, + encodeCameraPassthrough, +} from './webgpu/cameraPassthroughPipeline'; +export { + type CameraShaderBindings, + createCameraBindGroup, + createCameraShaderBindings, + type CreateCameraShaderBindingsOptions, +} from './webgpu/cameraShaderBindings'; +export { + type CameraTextureResolver, + createCameraTextureResolver, + resolveCameraTexture, +} from './webgpu/cameraTextureResolver'; +export { computeAspectFillCrop, computeSquareCrop, type FrameCrop, packFrameCropParams } from './webgpu/cropUtilities'; +export type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './webgpu/frameRenderContext'; +export { + assertWebGpuDeviceSupportsCameraImport, + getOutputSurfaceFormat, + getRequiredWebGpuCameraFeatures, +} from './webgpu/requiredFeatures'; +export { useCameraWebGpuDevice, type UseCameraWebGpuDeviceResult } from './webgpu/useCameraWebGpuDevice'; +export { + useVisionCameraWebGpuSource, + type UseVisionCameraWebGpuSourceOptions, + type UseVisionCameraWebGpuSourceResult, +} from './webgpu/useVisionCameraWebGpuSource'; diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts new file mode 100644 index 00000000..52685a09 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts @@ -0,0 +1,152 @@ +import { GPUBufferUsage, GPUShaderStage } from 'react-native-webgpu'; + +import { type CameraShaderBindings, createCameraBindGroup, createCameraShaderBindings } from './cameraShaderBindings'; +import { type FrameCrop, packFrameCropParams } from './cropUtilities'; +import { getOutputSurfaceFormat } from './requiredFeatures'; + +const CAMERA_BIND_GROUP_INDEX = 0; +const CROP_BIND_GROUP_INDEX = 1; + +const FRAME_CROP_BUFFER_BYTES = 40; + +// One oversized triangle covering the viewport; uv spans [0,1] top-left origin over the visible +// area. The fragment applies the FrameCropParams crop + orientation transform (same math as the +// charades composite) and samples the camera via the injected sampleCamera(). +function buildPassthroughShaderCode(cameraShaderCode: string, mirror: boolean): string { + const cropUvExpression = mirror ? 'vec2f(1.0 - screenUv.x, screenUv.y)' : 'screenUv'; + return /* wgsl */ `${cameraShaderCode} +struct FrameCropParams { + sourceSize: vec2u, + cropOrigin: vec2f, + cropSize: vec2f, + uvTransform: mat2x2f, +} + +@group(${CROP_BIND_GROUP_INDEX}) @binding(0) var cropParams: FrameCropParams; + +struct VertexOutput { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +} + +@vertex +fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { + var positions = array(vec2f(-1.0, -1.0), vec2f(3.0, -1.0), vec2f(-1.0, 3.0)); + let position = positions[vertexIndex]; + var output: VertexOutput; + output.position = vec4f(position, 0.0, 1.0); + output.uv = vec2f((position.x + 1.0) * 0.5, 1.0 - (position.y + 1.0) * 0.5); + return output; +} + +@fragment +fn fragmentMain(@location(0) screenUv: vec2f) -> @location(0) vec4f { + let cropUv = ${cropUvExpression}; + let sourcePixel = cropParams.cropOrigin + cropUv * cropParams.cropSize; + let sourceUv = sourcePixel / vec2f(cropParams.sourceSize); + let cameraUv = cropParams.uvTransform * (sourceUv - vec2f(0.5)) + vec2f(0.5); + return sampleCamera(cameraUv); +} +`; +} + +/** Options for {@link createCameraPassthroughPipeline}. */ +export interface CameraPassthroughPipelineOptions { + /** Render-target format. Defaults to {@link getOutputSurfaceFormat} (the Fishjam output surface). */ + outputFormat?: GPUTextureFormat; + /** Mirror the camera horizontally (the usual selfie self-view convention). Defaults to `false`. */ + mirror?: boolean; +} + +/** + * A ready-made full-screen camera→target render pipeline. Build once at setup with + * {@link createCameraPassthroughPipeline}; every field is safe to capture into the frame worklet. + * + * @group WebGPU + */ +export interface CameraPassthroughPipeline { + readonly pipeline: GPURenderPipeline; + /** The camera shader bindings the pipeline samples through (group 0). */ + readonly cameraShaderBindings: CameraShaderBindings; + /** Uniform buffer holding the packed FrameCropParams; written by {@link encodeCameraPassthrough}. */ + readonly cropParamsBuffer: GPUBuffer; + /** Static bind group with the crop uniform (group 1). */ + readonly cropBindGroup: GPUBindGroup; +} + +/** + * Builds the full-screen camera passthrough pipeline: crop/orientation via {@link FrameCrop}, + * platform-correct camera sampling, one triangle. Use it to publish the camera through the WebGPU + * tier with zero WGSL of your own, or as the base pass under your overlay passes. + * + * @group WebGPU + */ +export function createCameraPassthroughPipeline( + device: GPUDevice, + options: CameraPassthroughPipelineOptions = {}, +): CameraPassthroughPipeline { + const outputFormat = options.outputFormat ?? getOutputSurfaceFormat(); + const cameraShaderBindings = createCameraShaderBindings(device, { bindGroupIndex: CAMERA_BIND_GROUP_INDEX }); + + const cropBindGroupLayout = device.createBindGroupLayout({ + label: 'fishjam-camera-passthrough-crop', + entries: [{ binding: 0, visibility: GPUShaderStage.FRAGMENT, buffer: { type: 'uniform' } }], + }); + const cropParamsBuffer = device.createBuffer({ + label: 'fishjam-camera-passthrough-crop-params', + size: FRAME_CROP_BUFFER_BYTES, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + const cropBindGroup = device.createBindGroup({ + layout: cropBindGroupLayout, + entries: [{ binding: 0, resource: { buffer: cropParamsBuffer } }], + }); + + const shaderModule = device.createShaderModule({ + label: 'fishjam-camera-passthrough', + code: buildPassthroughShaderCode(cameraShaderBindings.shaderCode, options.mirror ?? false), + }); + const pipeline = device.createRenderPipeline({ + label: 'fishjam-camera-passthrough', + layout: device.createPipelineLayout({ + bindGroupLayouts: [cameraShaderBindings.bindGroupLayout, cropBindGroupLayout], + }), + vertex: { module: shaderModule, entryPoint: 'vertexMain' }, + fragment: { + module: shaderModule, + entryPoint: 'fragmentMain', + targets: [{ format: outputFormat }], + }, + primitive: { topology: 'triangle-list' }, + }); + + return { pipeline, cameraShaderBindings, cropParamsBuffer, cropBindGroup }; +} + +/** + * Encodes one full-screen pass drawing the camera into `outputView`, cropped per `crop`. + * Worklet-safe; call it inside your render callback, and encode any overlay passes after it on + * the same command encoder (with `loadOp: 'load'` so they draw on top). + * + * @group WebGPU + */ +export function encodeCameraPassthrough( + device: GPUDevice, + passthrough: CameraPassthroughPipeline, + cameraTexture: GPUExternalTexture, + outputView: GPUTextureView, + commandEncoder: GPUCommandEncoder, + crop: FrameCrop, +): void { + 'worklet'; + device.queue.writeBuffer(passthrough.cropParamsBuffer, 0, packFrameCropParams(crop)); + const cameraBindGroup = createCameraBindGroup(device, passthrough.cameraShaderBindings, cameraTexture); + const pass = commandEncoder.beginRenderPass({ + colorAttachments: [{ view: outputView, loadOp: 'clear', storeOp: 'store', clearValue: [0, 0, 0, 1] }], + }); + pass.setPipeline(passthrough.pipeline); + pass.setBindGroup(CAMERA_BIND_GROUP_INDEX, cameraBindGroup); + pass.setBindGroup(CROP_BIND_GROUP_INDEX, passthrough.cropBindGroup); + pass.draw(3); + pass.end(); +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts new file mode 100644 index 00000000..c3818f4e --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts @@ -0,0 +1,119 @@ +import { Platform } from 'react-native'; +import { GPUShaderStage } from 'react-native-webgpu'; + +// On Android, the camera arrives as an opaque YCbCr AHardwareBuffer and Dawn's Vulkan path forces +// an identity sampler conversion, so sampling the external texture returns RAW [Y, Cb, Cr] — the +// BT.709 limited-range decode below must run in-shader. iOS (NV12 IOSurface) samples as +// ready-to-use RGB. +const SAMPLE_CAMERA_BODY_ANDROID = /* wgsl */ ` + let rawSample = textureSampleBaseClampToEdge(fishjamCameraTexture, fishjamCameraSampler, uv); + let luma = rawSample.r - 0.0627451; + let chromaBlue = rawSample.g - 0.5; + let chromaRed = rawSample.b - 0.5; + let rgb = vec3f( + 1.164384 * luma + 1.792741 * chromaRed, + 1.164384 * luma - 0.213249 * chromaBlue - 0.532909 * chromaRed, + 1.164384 * luma + 2.112402 * chromaBlue, + ); + return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0);`; + +const SAMPLE_CAMERA_BODY_IOS = /* wgsl */ ` + return textureSampleBaseClampToEdge(fishjamCameraTexture, fishjamCameraSampler, uv);`; + +function buildCameraShaderCode(bindGroupIndex: number): string { + const body = Platform.OS === 'android' ? SAMPLE_CAMERA_BODY_ANDROID : SAMPLE_CAMERA_BODY_IOS; + return /* wgsl */ ` +@group(${bindGroupIndex}) @binding(0) var fishjamCameraTexture: texture_external; +@group(${bindGroupIndex}) @binding(1) var fishjamCameraSampler: sampler; + +fn sampleCamera(uv: vec2f) -> vec4f {${body} +} +`; +} + +/** Options for {@link createCameraShaderBindings}. */ +export interface CreateCameraShaderBindingsOptions { + /** Bind group index the camera bindings are declared at in {@link CameraShaderBindings.shaderCode}. Defaults to `0`. */ + bindGroupIndex?: number; +} + +/** + * Everything a fragment shader needs to sample the live camera. Build once at setup with + * {@link createCameraShaderBindings}; the fields are safe to capture into the frame worklet. + * + * @group WebGPU + */ +export interface CameraShaderBindings { + /** + * WGSL to prepend to your fragment shader. It declares the camera texture + sampler at + * {@link bindGroupIndex} and defines `sampleCamera(uv: vec2f) -> vec4f`, which returns upright + * RGB on both platforms (the Android in-shader YUV decode is included automatically). + */ + readonly shaderCode: string; + /** Layout of the camera bind group; place it at {@link bindGroupIndex} in your pipeline layout. */ + readonly bindGroupLayout: GPUBindGroupLayout; + /** The linear-filtering sampler bound at binding 1. */ + readonly sampler: GPUSampler; + /** The group index the bindings are declared at. */ + readonly bindGroupIndex: number; +} + +/** + * Builds the camera-sampling shader bindings against the device your pipelines use. Pass the + * result to the source hook's `cameraShaderBindings` option and the render context delivers a + * ready-made `cameraBindGroup` every frame — set it at {@link CameraShaderBindings.bindGroupIndex} + * and call `sampleCamera(uv)` in your fragment shader. + * + * ```ts + * const cameraBindings = createCameraShaderBindings(device); + * const module = device.createShaderModule({ code: cameraBindings.shaderCode + MY_FRAGMENT_WGSL }); + * ``` + * + * @group WebGPU + */ +export function createCameraShaderBindings( + device: GPUDevice, + options: CreateCameraShaderBindingsOptions = {}, +): CameraShaderBindings { + const bindGroupIndex = options.bindGroupIndex ?? 0; + const bindGroupLayout = device.createBindGroupLayout({ + label: 'fishjam-camera-shader-bindings', + entries: [ + { binding: 0, visibility: GPUShaderStage.FRAGMENT, externalTexture: {} }, + { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: {} }, + ], + }); + const sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' }); + return { + shaderCode: buildCameraShaderCode(bindGroupIndex), + bindGroupLayout, + sampler, + bindGroupIndex, + }; +} + +/** + * Builds the per-frame bind group for the live camera texture. The source hook already does this + * for you when you pass `cameraShaderBindings` in its options (see the render context's + * `cameraBindGroup`); call it yourself only for advanced multi-layout setups. Worklet-safe. + * + * A camera texture expires when the frame ends, so a bind group referencing it must be rebuilt + * every frame — never cache the result. + * + * @group WebGPU + */ +export function createCameraBindGroup( + device: GPUDevice, + cameraShaderBindings: CameraShaderBindings, + cameraTexture: GPUExternalTexture, +): GPUBindGroup { + 'worklet'; + return device.createBindGroup({ + label: 'fishjam-camera-frame', + layout: cameraShaderBindings.bindGroupLayout, + entries: [ + { binding: 0, resource: cameraTexture }, + { binding: 1, resource: cameraShaderBindings.sampler }, + ], + }); +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts new file mode 100644 index 00000000..e095736c --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts @@ -0,0 +1,69 @@ +import { GPUTextureUsage } from 'react-native-webgpu'; + +import { createCameraPassthroughPipeline, encodeCameraPassthrough } from './cameraPassthroughPipeline'; +import { computeAspectFillCrop } from './cropUtilities'; + +/** + * An owned `rgba8unorm` texture the camera frame is resolved into each frame — for pipelines that + * want a plain `texture_2d` camera instead of `texture_external`. Build once at setup with + * {@link createCameraTextureResolver}; every field is safe to capture into the frame worklet. + * + * Prefer sampling the camera directly via {@link createCameraShaderBindings} when you can: the + * resolver costs one extra render pass per frame. + * + * @group WebGPU + */ +export interface CameraTextureResolver { + /** The resolved camera texture (`rgba8unorm`, sampled + render-attachment usage). */ + readonly texture: GPUTexture; + /** A reusable default view of {@link texture}. */ + readonly view: GPUTextureView; + readonly width: number; + readonly height: number; + /** @internal The pass used to resolve into {@link texture}. */ + readonly resolvePass: ReturnType; +} + +/** + * Creates a {@link CameraTextureResolver} with an owned `rgba8unorm` texture of the given size. + * + * @group WebGPU + */ +export function createCameraTextureResolver( + device: GPUDevice, + size: { width: number; height: number }, +): CameraTextureResolver { + const texture = device.createTexture({ + label: 'fishjam-resolved-camera', + format: 'rgba8unorm', + size: [size.width, size.height], + usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.RENDER_ATTACHMENT, + }); + return { + texture, + view: texture.createView(), + width: size.width, + height: size.height, + resolvePass: createCameraPassthroughPipeline(device, { outputFormat: 'rgba8unorm' }), + }; +} + +/** + * Encodes one pass resolving the live camera texture into `resolver.texture`, aspect-filled to + * the resolver's size (platform YUV decode included). Worklet-safe; call it inside your render + * callback before the passes that sample `resolver.texture`. + * + * @group WebGPU + */ +export function resolveCameraTexture( + device: GPUDevice, + resolver: CameraTextureResolver, + cameraTexture: GPUExternalTexture, + cameraWidth: number, + cameraHeight: number, + commandEncoder: GPUCommandEncoder, +): void { + 'worklet'; + const crop = computeAspectFillCrop(cameraWidth, cameraHeight, resolver.width / resolver.height); + encodeCameraPassthrough(device, resolver.resolvePass, cameraTexture, resolver.view, commandEncoder, crop); +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts b/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts new file mode 100644 index 00000000..67d1eb08 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts @@ -0,0 +1,131 @@ +/** + * Crop math + a worklet-safe byte packer for the FrameCropParams uniform used by + * {@link createCameraPassthroughPipeline} (and reusable by your own shaders). + * + * FrameCropParams byte layout (40 bytes), matching the WGSL struct + * `{ sourceSize: vec2u, cropOrigin: vec2f, cropSize: vec2f, uvTransform: mat2x2f }`: + * + * [0] u32 sourceSize.x [1] u32 sourceSize.y + * [2] f32 cropOrigin.x [3] f32 cropOrigin.y + * [4] f32 cropSize.x [5] f32 cropSize.y + * [6..9] f32 uvTransform, column-major (m00, m01, m10, m11) + */ + +/** + * A center-crop of a source frame plus a UV-space orientation transform, in source pixels. + * Produce one with {@link computeAspectFillCrop} or {@link computeSquareCrop}. + * + * @group WebGPU + */ +export interface FrameCrop { + sourceWidth: number; + sourceHeight: number; + cropOriginX: number; + cropOriginY: number; + cropSizeX: number; + cropSizeY: number; + /** uvTransform matrix scalars in (m00, m01, m10, m11) order; identity when omitted at compute time. */ + uv00: number; + uv01: number; + uv10: number; + uv11: number; +} + +const FRAME_CROP_BYTES = 40; + +/** + * Packs a {@link FrameCrop} into the 40-byte FrameCropParams uniform layout (see the module doc). + * Worklet-safe; upload the result with `device.queue.writeBuffer(buffer, 0, bytes)`. + * + * @group WebGPU + */ +export function packFrameCropParams(crop: FrameCrop): ArrayBuffer { + 'worklet'; + const buffer = new ArrayBuffer(FRAME_CROP_BYTES); + const asUint32 = new Uint32Array(buffer); + const asFloat32 = new Float32Array(buffer); + asUint32[0] = crop.sourceWidth >>> 0; + asUint32[1] = crop.sourceHeight >>> 0; + asFloat32[2] = crop.cropOriginX; + asFloat32[3] = crop.cropOriginY; + asFloat32[4] = crop.cropSizeX; + asFloat32[5] = crop.cropSizeY; + asFloat32[6] = crop.uv00; + asFloat32[7] = crop.uv01; + asFloat32[8] = crop.uv10; + asFloat32[9] = crop.uv11; + return buffer; +} + +/** + * Center-crop of a (sourceWidth × sourceHeight) frame to the target aspect ratio (width/height). + * A no-op full-frame crop when the source already has the target aspect. Worklet-safe. + * + * Feed it the upright camera size from the render context: + * `computeAspectFillCrop(context.cameraWidth, context.cameraHeight, context.outputWidth / context.outputHeight)`. + * + * The optional `uv00..uv11` scalars form a UV-space transform applied around the frame center + * (identity by default). + * + * @group WebGPU + */ +export function computeAspectFillCrop( + sourceWidth: number, + sourceHeight: number, + targetAspect: number, + uv00: number = 1, + uv01: number = 0, + uv10: number = 0, + uv11: number = 1, +): FrameCrop { + 'worklet'; + let cropWidth = sourceWidth; + let cropHeight = sourceHeight; + if (sourceWidth / sourceHeight > targetAspect) { + cropWidth = sourceHeight * targetAspect; + } else { + cropHeight = sourceWidth / targetAspect; + } + return { + sourceWidth, + sourceHeight, + cropOriginX: Math.floor((sourceWidth - cropWidth) / 2), + cropOriginY: Math.floor((sourceHeight - cropHeight) / 2), + cropSizeX: cropWidth, + cropSizeY: cropHeight, + uv00, + uv01, + uv10, + uv11, + }; +} + +/** + * Square center-crop of a (sourceWidth × sourceHeight) frame — the usual shape for square model + * inputs. Worklet-safe. The optional `uv00..uv11` scalars are as in {@link computeAspectFillCrop}. + * + * @group WebGPU + */ +export function computeSquareCrop( + sourceWidth: number, + sourceHeight: number, + uv00: number = 1, + uv01: number = 0, + uv10: number = 0, + uv11: number = 1, +): FrameCrop { + 'worklet'; + const size = Math.min(sourceWidth, sourceHeight); + return { + sourceWidth, + sourceHeight, + cropOriginX: Math.floor((sourceWidth - size) / 2), + cropOriginY: Math.floor((sourceHeight - size) / 2), + cropSizeX: size, + cropSizeY: size, + uv00, + uv01, + uv10, + uv11, + }; +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts b/packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts new file mode 100644 index 00000000..624c935e --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts @@ -0,0 +1,61 @@ +/** + * Everything your render callback needs for one frame. Handed to the function you pass to + * `render(...)` inside the source hook's `onFrame` worklet; every field is valid only until that + * function returns. + * + * @group WebGPU + */ +export interface WebGpuFrameRenderContext { + /** The device in use (the shared device, or your override). */ + device: GPUDevice; + /** Shortcut for `device.queue`. */ + queue: GPUQueue; + /** + * The command encoder for this frame. Encode your render/compute passes into it — the hook + * submits it (a single submit) and delivers the frame after your callback returns. Do not call + * `queue.submit` yourself for work targeting {@link outputTexture}. + */ + commandEncoder: GPUCommandEncoder; + /** + * The live camera frame as a `texture_external`, already rotated upright (and mirrored per the + * camera's mirroring). On Android it samples as raw YCbCr — sample it through + * `createCameraShaderBindings` (or the ready-made {@link cameraBindGroup}) for + * platform-correct RGB. + */ + cameraTexture: GPUExternalTexture; + /** + * Ready-made bind group for {@link cameraTexture}, present when the hook's + * `cameraShaderBindings` option was provided: set it at the bindings' group index and + * `sampleCamera(uv)` works. + */ + cameraBindGroup?: GPUBindGroup; + /** The output texture your passes draw into; its content becomes the published video frame. */ + outputTexture: GPUTexture; + /** + * A cached, reusable default `createView()` of {@link outputTexture}. Prefer this over calling + * `outputTexture.createView()` yourself every frame: `GPUTextureView` has no `destroy()`/`release()` + * in react-native-webgpu, so a per-frame view accumulates native (malloc) wrappers on the frame + * runtime until GC — a steady leak in a render loop. The output textures are a small fixed pool, so + * the hook builds one view per slot and reuses it. + */ + outputView: GPUTextureView; + /** Size of {@link outputTexture}, in pixels. */ + outputWidth: number; + outputHeight: number; + /** + * Upright (post-rotation) camera frame size, in pixels — feed it to `computeAspectFillCrop` + * together with the output aspect. + */ + cameraWidth: number; + cameraHeight: number; + /** Whether the camera feed is mirrored (front cameras usually are). */ + cameraIsMirrored: boolean; +} + +/** + * The `render` function handed to the source hook's `onFrame` worklet. Call it at most once per + * frame with your encode function; skipping it drops the frame (nothing is published for it). + * + * @group WebGPU + */ +export type WebGpuFrameRenderFunction = (encode: (context: WebGpuFrameRenderContext) => void) => void; diff --git a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts new file mode 100644 index 00000000..f56288ba --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts @@ -0,0 +1,53 @@ +import { Platform } from 'react-native'; + +// rnwebgpu/native-texture gates the zero-copy shared-surface + external-texture path. +// The camera-import feature differs by platform: iOS delivers biplanar NV12 IOSurfaces +// (dawn-multi-planar-formats); Android delivers YUV_420 AHardwareBuffers imported through an +// external YCbCr Vulkan sampler (opaque-ycbcr-android-for-external-texture). Without the right +// one, importing the camera fails on every frame on a real device. +const REQUIRED_FEATURES: GPUFeatureName[] = + Platform.OS === 'android' + ? ['rnwebgpu/native-texture' as GPUFeatureName, 'opaque-ycbcr-android-for-external-texture' as GPUFeatureName] + : ['rnwebgpu/native-texture' as GPUFeatureName, 'dawn-multi-planar-formats' as GPUFeatureName]; + +/** + * The GPU features a device must have to import camera frames and Fishjam output surfaces on this + * platform. {@link useCameraWebGpuDevice} requests them for you; pass them yourself as + * `requiredFeatures` in `GPUDeviceDescriptor` when you bring your own device. + * + * @group WebGPU + */ +export function getRequiredWebGpuCameraFeatures(): GPUFeatureName[] { + return [...REQUIRED_FEATURES]; +} + +/** + * Throws a descriptive error when `device` is missing any feature required to import camera + * frames or Fishjam output surfaces on this platform. Called automatically on devices passed as + * an override; call it yourself to validate a device early. + * + * @group WebGPU + */ +export function assertWebGpuDeviceSupportsCameraImport(device: GPUDevice): void { + const missingFeatures = REQUIRED_FEATURES.filter((feature) => !device.features.has(feature)); + if (missingFeatures.length === 0) { + return; + } + throw new Error( + `This GPUDevice cannot import camera frames on ${Platform.OS}: it is missing the ` + + `${missingFeatures.map((feature) => `'${feature}'`).join(', ')} feature(s). ` + + `Request your device with getRequiredWebGpuCameraFeatures() in GPUDeviceDescriptor.requiredFeatures, ` + + `or omit the device option to use the shared device from useCameraWebGpuDevice().`, + ); +} + +/** + * The pixel format of Fishjam output surfaces on this platform: `'rgba8unorm'` on Android + * (AHardwareBuffer), `'bgra8unorm'` on iOS (IOSurface). Use it as the render-target format of any + * pipeline that draws into the output texture — a mismatched format renders wrong or black. + * + * @group WebGPU + */ +export function getOutputSurfaceFormat(): GPUTextureFormat { + return Platform.OS === 'android' ? 'rgba8unorm' : 'bgra8unorm'; +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts new file mode 100644 index 00000000..0178ea99 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts @@ -0,0 +1,110 @@ +import { useEffect, useMemo, useState } from 'react'; + +import { assertWebGpuDeviceSupportsCameraImport, getRequiredWebGpuCameraFeatures } from './requiredFeatures'; +import { getWebGpuRuntime } from './webGpuRuntime'; + +// One app-wide device: every hook instance shares it, so per-device resources (pipelines, +// shared-texture imports) built by different callers stay valid for each other. Cleared on +// device loss or acquisition failure so a later caller retries. +let sharedDevicePromise: Promise | null = null; + +async function acquireSharedCameraWebGpuDevice(): Promise { + getWebGpuRuntime(); + const adapter = await navigator.gpu.requestAdapter(); + if (adapter == null) { + throw new Error('WebGPU is unavailable: navigator.gpu.requestAdapter() returned no adapter.'); + } + const device = await adapter.requestDevice({ + requiredFeatures: getRequiredWebGpuCameraFeatures(), + }); + device.lost + .then(() => { + sharedDevicePromise = null; + }) + .catch(() => { + sharedDevicePromise = null; + }); + return device; +} + +function getSharedCameraWebGpuDevice(): Promise { + if (sharedDevicePromise == null) { + sharedDevicePromise = acquireSharedCameraWebGpuDevice().catch((cause: unknown) => { + sharedDevicePromise = null; + throw cause; + }); + } + return sharedDevicePromise; +} + +/** Result of {@link useCameraWebGpuDevice}. */ +export interface UseCameraWebGpuDeviceResult { + /** The shared GPUDevice; `null` until acquisition resolves. */ + device: GPUDevice | null; + /** Acquisition failure (no adapter, missing platform features), if any. */ + error: Error | null; +} + +/** + * The app-wide GPUDevice used for camera work, configured with + * {@link getRequiredWebGpuCameraFeatures}. All callers share one device, so pipelines you build + * against it work with the textures the source hook hands your render callback. + * + * Build your pipelines once the device arrives: + * ```tsx + * const { device } = useCameraWebGpuDevice(); + * const effect = useMemo(() => (device ? buildMyEffect(device) : null), [device]); + * ``` + * + * @group WebGPU + */ +export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult { + const [result, setResult] = useState({ device: null, error: null }); + + useEffect(() => { + let cancelled = false; + getSharedCameraWebGpuDevice() + .then((device) => { + if (!cancelled) { + setResult({ device, error: null }); + } + }) + .catch((cause: unknown) => { + if (!cancelled) { + setResult({ device: null, error: cause instanceof Error ? cause : new Error(String(cause)) }); + } + }); + return () => { + cancelled = true; + }; + }, []); + + return result; +} + +/** + * Device resolution for the source hook: the user-provided override (validated) when present, + * otherwise the shared device. Always called, so hook order stays stable either way. + */ +export function useCameraWebGpuDeviceWithOverride(override: GPUDevice | undefined): UseCameraWebGpuDeviceResult { + const shared = useCameraWebGpuDevice(); + + const overrideValidationError = useMemo(() => { + if (override == null) { + return null; + } + try { + assertWebGpuDeviceSupportsCameraImport(override); + return null; + } catch (cause) { + return cause instanceof Error ? cause : new Error(String(cause)); + } + }, [override]); + + if (override != null) { + return overrideValidationError != null + ? { device: null, error: overrideValidationError } + : { device: override, error: null }; + } + return shared; +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts new file mode 100644 index 00000000..a17c74d6 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -0,0 +1,325 @@ +import { type MediaStream, pushFrame } from '@fishjam-cloud/react-native-webrtc'; +import { useMemo } from 'react'; +import { + type CameraFrameOutput, + type Frame, + type FrameDroppedReason, + type FrameOutputOptions, + useFrameOutput, +} from 'react-native-vision-camera'; +import { type GPUSharedTextureMemory, GPUTextureUsage } from 'react-native-webgpu'; + +import { createFrameTimestampState, nextFrameTimestampNanoseconds } from '../frameTimestamp'; +import { useManagedCustomSource } from '../internal/useManagedCustomSource'; +import { useManagedPooledTrack } from '../internal/useManagedPooledTrack'; +import { rotationDegreesFromOrientation } from '../orientation'; +import { type CameraShaderBindings, createCameraBindGroup } from './cameraShaderBindings'; +import type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './frameRenderContext'; +import { getOutputSurfaceFormat } from './requiredFeatures'; +import { useCameraWebGpuDeviceWithOverride } from './useCameraWebGpuDevice'; +import { getWebGpuRuntime } from './webGpuRuntime'; + +const DEFAULT_POOL_SIZE = 3; +const DEFAULT_FRAME_INTERVAL_NANOSECONDS = 33_333_333; // 30 fps fallback cadence + +/** + * Options for {@link useVisionCameraWebGpuSource}. Also accepts every VisionCamera frame-output + * option except `pixelFormat`, which the hook forces to `'native'` (the zero-copy camera-import + * path requires it). + */ +export interface UseVisionCameraWebGpuSourceOptions extends Partial> { + /** + * Whether the source is live. While `false`, no track or surface pool exists and nothing is + * published — the declarative sibling of VisionCamera's `isActive`. Defaults to `true`. + */ + enabled?: boolean; + /** + * How receivers see the published video — same semantics as `useCustomSource`: + * `'camera'` publishes it as the peer's camera track (a "virtual camera"), `'customVideo'` + * (the default) as a separate custom video track. + */ + videoType?: 'camera' | 'customVideo'; + /** Width of the published video, in pixels. */ + width: number; + /** Height of the published video, in pixels. */ + height: number; + /** + * Number of in-flight output surfaces (a pushed frame may still be encoding while the next one + * is drawn). Defaults to `3`. + */ + poolSize?: number; + /** + * Bring your own GPUDevice instead of the shared one from `useCameraWebGpuDevice`. It is + * validated against the required camera-import features; a device missing any of them surfaces + * a descriptive `error` instead of failing per frame. + */ + device?: GPUDevice; + /** + * Camera shader bindings built with `createCameraShaderBindings`. When set, the render context + * carries a ready-made `cameraBindGroup` for the live camera texture every frame. + */ + cameraShaderBindings?: CameraShaderBindings; + /** + * Worklet called for every camera frame. Call `render(...)` at most once to draw this frame's + * output; skipping it drops the frame (nothing is published for it). After `render(...)` + * returns you may keep using `frame` (for example run inference) — but only until this + * callback returns, when the hook releases the frame. Do not retain it. Keep the function's + * identity stable (`useCallback` or module scope). + */ + onFrame: (frame: Frame, render: WebGpuFrameRenderFunction) => void; + /** Called whenever the camera pipeline drops a frame; forwarded to VisionCamera. */ + onFrameDropped?: (reason: FrameDroppedReason) => void; + /** + * Fallback spacing between published frames, in nanoseconds, used only when a camera frame + * carries no usable timestamp. Defaults to 33,333,333 (30 fps). + */ + frameIntervalNanoseconds?: number; +} + +/** Result of {@link useVisionCameraWebGpuSource}. */ +export interface UseVisionCameraWebGpuSourceResult { + /** + * The VisionCamera frame output driving this source. Plug it into your camera session: + * `useCamera({ device, isActive, outputs: [frameOutput] })`. + */ + frameOutput: CameraFrameOutput; + /** + * The published stream — render it with `RTCView` for a self-view. `null` until the underlying + * track is ready (creation is asynchronous). + */ + stream: MediaStream | null; + /** The GPUDevice in use — build your pipelines against it. `null` until acquired. */ + device: GPUDevice | null; + /** Track creation, publishing, or device acquisition failure, if any. */ + error: Error | null; +} + +/** + * Publishes WebGPU-rendered video to Fishjam, fed by your VisionCamera feed. + * + * A sibling of `useCustomSource`: the hook creates the video track (and its pool of output + * surfaces), publishes it under `sourceId`, and cleans everything up on unmount. Every camera + * frame reaches your `onFrame` worklet, where calling `render(...)` hands you the live camera as + * a GPU texture plus an output texture to draw into — what you draw is what peers receive. + * Everything else is handled for you: output-surface management, GPU synchronization with the + * video encoder, timestamps, rotation, and frame lifetimes. Must be used under `FishjamProvider`. + * + * ```tsx + * const { frameOutput, stream, device } = useVisionCameraWebGpuSource('my-camera', { + * videoType: 'camera', + * width: 720, + * height: 1280, + * cameraShaderBindings: effect?.cameraBindings, + * onFrame, + * }); + * useVisionCamera({ device: cameraDevice, isActive: true, outputs: [frameOutput] }); + * ``` + * + * To publish the camera unmodified (no rendering), use `useVisionCameraSource` from the package + * root instead. For a ready-made camera→output pass to build on, see + * `createCameraPassthroughPipeline`. + * + * @param sourceId Identifies this source among the peer's tracks, like in `useCustomSource`. + * @param options See {@link UseVisionCameraWebGpuSourceOptions}. + * @group Hooks + */ +export function useVisionCameraWebGpuSource( + sourceId: SourceId, + options: UseVisionCameraWebGpuSourceOptions, +): UseVisionCameraWebGpuSourceResult { + const { + enabled = true, + videoType, + width, + height, + poolSize = DEFAULT_POOL_SIZE, + device: deviceOverride, + cameraShaderBindings, + onFrame: userOnFrame, + onFrameDropped, + frameIntervalNanoseconds = DEFAULT_FRAME_INTERVAL_NANOSECONDS, + ...frameOutputOptions + } = options; + + const { device, error: deviceError } = useCameraWebGpuDeviceWithOverride(deviceOverride); + const { + track, + stream, + bufferDescriptors, + error: trackError, + } = useManagedPooledTrack(enabled, width, height, poolSize); + useManagedCustomSource(sourceId, videoType, stream); + + const runtime = getWebGpuRuntime(); + const outputSurfaceFormat = getOutputSurfaceFormat(); + // Captured as a plain number: the worklet must not close over the GPUTextureUsage namespace. + const renderAttachmentUsage = GPUTextureUsage.RENDER_ATTACHMENT; + + const timestampState = useMemo(() => createFrameTimestampState(), []); + + // Worklet-side per-source state. Plain boxes copied into the worklet closure on (re-)creation: + // the frame thread's copy carries the pool cursor and the per-slot shared-surface imports + // (importSharedTextureMemory + begin/endAccess must all run on the frame runtime). Keyed by + // [track, device] so a new track or device starts from an empty cache and never touches stale + // imports. Imports abandoned by a replaced closure are released by the frame runtime's GC; the + // deterministic alternative (import + destroy every frame) costs an import per frame — switch + // to it if leak measurements ever demand. + const workletState = useMemo(() => { + void track; + void device; + return { + poolCursor: 0, + importedByIndex: {} as Record< + number, + { memory: GPUSharedTextureMemory; texture: GPUTexture; view: GPUTextureView } + >, + }; + }, [track, device]); + + const handleFrame = useMemo(() => { + return (frame: Frame) => { + 'worklet'; + try { + if (track == null || bufferDescriptors == null || device == null) { + return; + } + const nativeBuffer = frame.getNativeBuffer(); + try { + const videoFrame = runtime.createVideoFrameFromNativeBuffer(nativeBuffer.pointer); + try { + const rotationDegrees = rotationDegreesFromOrientation(frame.orientation); + const isRotatedQuarterTurn = rotationDegrees === 90 || rotationDegrees === 270; + const cameraWidth = isRotatedQuarterTurn ? videoFrame.height : videoFrame.width; + const cameraHeight = isRotatedQuarterTurn ? videoFrame.width : videoFrame.height; + + const cameraTexture = device.importExternalTexture({ + // react-native-webgpu accepts its NativeVideoFrame here at runtime, but its type + // declarations don't widen the descriptor's `source`, so cast. + source: videoFrame as unknown as VideoFrame, + label: 'fishjam-camera-frame', + rotation: rotationDegrees, + mirrored: frame.isMirrored, + }); + try { + const cameraBindGroup = + cameraShaderBindings != null + ? createCameraBindGroup(device, cameraShaderBindings, cameraTexture) + : undefined; + + let rendered = false; + const render: WebGpuFrameRenderFunction = (encode) => { + 'worklet'; + if (rendered) { + throw new Error('useVisionCameraWebGpuSource: render() may only be called once per frame.'); + } + rendered = true; + + const cursor = workletState.poolCursor; + workletState.poolCursor = (cursor + 1) % bufferDescriptors.length; + const descriptor = bufferDescriptors[cursor]; + + let imported = workletState.importedByIndex[descriptor.index]; + if (imported == null) { + const memory = device.importSharedTextureMemory({ handle: descriptor.surfaceHandle }); + const texture = memory.createTexture({ + format: outputSurfaceFormat, + size: [descriptor.width, descriptor.height], + usage: renderAttachmentUsage, + }); + // Build the output view ONCE per pool slot and reuse it every frame: a + // GPUTextureView has no release API, so a per-frame createView() leaks native + // wrappers on the frame runtime until GC. + imported = { memory, texture, view: texture.createView() }; + workletState.importedByIndex[descriptor.index] = imported; + } + + imported.memory.beginAccess(imported.texture, false); + let accessResult; + try { + const commandEncoder = device.createCommandEncoder(); + const context: WebGpuFrameRenderContext = { + device, + queue: device.queue, + commandEncoder, + cameraTexture, + cameraBindGroup, + outputTexture: imported.texture, + outputView: imported.view, + outputWidth: descriptor.width, + outputHeight: descriptor.height, + cameraWidth, + cameraHeight, + cameraIsMirrored: frame.isMirrored, + }; + encode(context); + device.queue.submit([commandEncoder.finish()]); + } finally { + // Always end the access scope — leaving a slot acquired after a throw would + // poison it for every later frame. + accessResult = imported.memory.endAccess(imported.texture); + } + + const fenceState = accessResult.fences[0]; + const timestampNanoseconds = nextFrameTimestampNanoseconds( + timestampState, + frame.timestamp, + frameIntervalNanoseconds, + ); + + // Push directly from the worklet — native retains the fence synchronously here, + // so no JS-side fence retention is needed. Rotation is 0: the camera was already + // rotated upright at import time. + pushFrame(track, { + bufferIndex: descriptor.index, + timestampNs: timestampNanoseconds, + rotation: 0, + ...(fenceState != null + ? { fence: { handle: fenceState.fence.export().handle, signaledValue: fenceState.signaledValue } } + : {}), + }); + }; + + userOnFrame(frame, render); + } finally { + // End the camera texture's access window now — waiting for GC would starve the + // camera frame buffer pool. + cameraTexture.destroy(); + } + } finally { + videoFrame.release(); + } + } finally { + nativeBuffer.release(); + } + } catch (cause) { + console.warn('useVisionCameraWebGpuSource: processing a camera frame failed: ' + String(cause)); + } finally { + frame.dispose(); + } + }; + }, [ + track, + bufferDescriptors, + device, + runtime, + cameraShaderBindings, + userOnFrame, + workletState, + timestampState, + outputSurfaceFormat, + renderAttachmentUsage, + frameIntervalNanoseconds, + ]); + + const frameOutput = useFrameOutput({ + dropFramesWhileBusy: true, + ...frameOutputOptions, + // 'native' is required for the zero-copy camera import; VisionCamera converts the frames + // (YUV→RGB, rotation) with any other format, and iOS Simulators cannot import the result. + pixelFormat: 'native', + onFrame: handleFrame, + onFrameDropped, + }); + + return { frameOutput, stream, device, error: deviceError ?? trackError }; +} diff --git a/packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts b/packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts new file mode 100644 index 00000000..a6eb3009 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts @@ -0,0 +1,21 @@ +import { WebGPUModule } from 'react-native-webgpu'; + +/** The react-native-webgpu runtime singleton (the `RNWebGPU` JSI global). */ +export type WebGpuRuntime = typeof RNWebGPU; + +/** + * Returns the react-native-webgpu runtime singleton, self-healing when the `RNWebGPU` global is + * missing on the calling runtime. The library binds the global once at module load, but that + * binding can be absent after a crash-recovery runtime reload or an install-ordering flake; + * `WebGPUModule.install()` binds it to the calling runtime and is safe to repeat. + */ +export function getWebGpuRuntime(): WebGpuRuntime { + if (typeof RNWebGPU === 'undefined') { + const installedOk = WebGPUModule.install(); + console.warn( + `react-native-vision-camera-source: the RNWebGPU global was missing on this runtime; ` + + `re-ran install() -> ${String(installedOk)}`, + ); + } + return RNWebGPU; +} diff --git a/packages/react-native-vision-camera-source/tsconfig.json b/packages/react-native-vision-camera-source/tsconfig.json new file mode 100644 index 00000000..75a4cfb2 --- /dev/null +++ b/packages/react-native-vision-camera-source/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "allowJs": true, + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist/", + "skipLibCheck": true, + "strict": true, + "target": "ESNext", + "lib": ["es2020", "DOM", "DOM.Iterable"], + "jsx": "react-jsx" + }, + "include": ["src/**/*"] +} diff --git a/packages/react-native-vision-camera-source/typedoc.json b/packages/react-native-vision-camera-source/typedoc.json new file mode 100644 index 00000000..b14f5ae9 --- /dev/null +++ b/packages/react-native-vision-camera-source/typedoc.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "includeVersion": true, + "entryPoints": ["src/index.ts", "src/webgpu.ts"], + "readme": "none" +} diff --git a/packages/react-native-webrtc b/packages/react-native-webrtc index 6b994c06..140df716 160000 --- a/packages/react-native-webrtc +++ b/packages/react-native-webrtc @@ -1 +1 @@ -Subproject commit 6b994c06a0065ef7e4c8f0169fb95a15303b3266 +Subproject commit 140df716ec61070d5891d74decc6df4c91ce3a14 diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index fab0e6f8..3b889f52 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -78,11 +78,14 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { - if (!this.connectionManager) { - this.connectionManager = new ConnectionManager(this.proposedIceServers); - } - - return this.connectionManager.createDataChannel(label, init); + // Must go through createNewConnection (same as onOfferData): a bare + // `new ConnectionManager()` here leaves the connection without ICE + // listeners, track managers, or the commands queue — and since + // onOfferData skips setup when a connection already exists, tracks + // added later are silently never published. + const connectionManager = this.connectionManager ?? this.createNewConnection(); + + return connectionManager.createDataChannel(label, init); }; const triggerRenegotiationFn = () => { diff --git a/release-automation/bump-version.sh b/release-automation/bump-version.sh index 7bd48677..ebee20cb 100755 --- a/release-automation/bump-version.sh +++ b/release-automation/bump-version.sh @@ -59,6 +59,9 @@ echo "Updated react-client to $VERSION" corepack yarn workspace @fishjam-cloud/react-native-client version "$VERSION" echo "Updated react-native-client to $VERSION" +corepack yarn workspace @fishjam-cloud/react-native-vision-camera-source version "$VERSION" +echo "Updated react-native-vision-camera-source to $VERSION" + # Run proto generation if corepack yarn gen:proto; then echo "Protos generated." diff --git a/yarn.lock b/yarn.lock index 105cd877..f6348fb3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -279,7 +279,7 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.29.7": +"@babel/generator@npm:^7.29.1, @babel/generator@npm:^7.29.7": version: 7.29.7 resolution: "@babel/generator@npm:7.29.7" dependencies: @@ -2727,7 +2727,7 @@ __metadata: languageName: node linkType: hard -"@babel/preset-typescript@npm:^7.16.7": +"@babel/preset-typescript@npm:^7.16.7, @babel/preset-typescript@npm:^7.28.5": version: 7.29.7 resolution: "@babel/preset-typescript@npm:7.29.7" dependencies: @@ -4553,6 +4553,39 @@ __metadata: languageName: unknown linkType: soft +"@fishjam-cloud/react-native-vision-camera-source@workspace:packages/react-native-vision-camera-source": + version: 0.0.0-use.local + resolution: "@fishjam-cloud/react-native-vision-camera-source@workspace:packages/react-native-vision-camera-source" + dependencies: + "@fishjam-cloud/react-native-client": "workspace:*" + "@fishjam-cloud/react-native-webrtc": "workspace:*" + "@types/react": "npm:19.2.14" + "@webgpu/types": "npm:0.1.65" + eslint-config-expo: "npm:~9.2.0" + eslint-import-resolver-typescript: "npm:^3.10.1" + eslint-plugin-prettier: "npm:^5.5.1" + react: "npm:19.2.3" + react-native: "npm:0.85.3" + react-native-nitro-modules: "npm:0.35.7" + react-native-vision-camera: "npm:5.0.10" + react-native-vision-camera-worklets: "npm:5.0.10" + react-native-webgpu: "npm:0.5.15" + react-native-worklets: "npm:>=0.9.0" + typescript: "npm:^5.8.3" + peerDependencies: + "@fishjam-cloud/react-native-client": "workspace:*" + "@fishjam-cloud/react-native-webrtc": "workspace:*" + react: "*" + react-native: "*" + react-native-vision-camera: ^5.0.0 + react-native-vision-camera-worklets: ^5.0.0 + react-native-webgpu: ">=0.5.15" + peerDependenciesMeta: + react-native-webgpu: + optional: true + languageName: unknown + linkType: soft + "@fishjam-cloud/react-native-webrtc@workspace:*, @fishjam-cloud/react-native-webrtc@workspace:packages/react-native-webrtc": version: 0.0.0-use.local resolution: "@fishjam-cloud/react-native-webrtc@workspace:packages/react-native-webrtc" @@ -5717,6 +5750,13 @@ __metadata: languageName: node linkType: hard +"@react-native/assets-registry@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/assets-registry@npm:0.85.3" + checksum: 10c0/c926848dae180940c19a68df42783900474a010944fc2f31f1b9a61ec722e6b4ead76943a560df5679d64d4784e7e945fd9f6d60f25555fdd1eb59c4326b8c29 + languageName: node + linkType: hard + "@react-native/babel-plugin-codegen@npm:0.81.5": version: 0.81.5 resolution: "@react-native/babel-plugin-codegen@npm:0.81.5" @@ -5881,6 +5921,23 @@ __metadata: languageName: node linkType: hard +"@react-native/codegen@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/codegen@npm:0.85.3" + dependencies: + "@babel/core": "npm:^7.25.2" + "@babel/parser": "npm:^7.29.0" + hermes-parser: "npm:0.33.3" + invariant: "npm:^2.2.4" + nullthrows: "npm:^1.1.1" + tinyglobby: "npm:^0.2.15" + yargs: "npm:^17.6.2" + peerDependencies: + "@babel/core": "*" + checksum: 10c0/3d6e08564c3436fcbd450c279b6755768ddfa47a534455c3077c222d5aa0a77270682afe933aa8d84bcd952b1a43107d25220d17c83f4f003fd338846a33ea4c + languageName: node + linkType: hard + "@react-native/community-cli-plugin@npm:0.81.5": version: 0.81.5 resolution: "@react-native/community-cli-plugin@npm:0.81.5" @@ -5904,6 +5961,29 @@ __metadata: languageName: node linkType: hard +"@react-native/community-cli-plugin@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/community-cli-plugin@npm:0.85.3" + dependencies: + "@react-native/dev-middleware": "npm:0.85.3" + debug: "npm:^4.4.0" + invariant: "npm:^2.2.4" + metro: "npm:^0.84.3" + metro-config: "npm:^0.84.3" + metro-core: "npm:^0.84.3" + semver: "npm:^7.1.3" + peerDependencies: + "@react-native-community/cli": "*" + "@react-native/metro-config": 0.85.3 + peerDependenciesMeta: + "@react-native-community/cli": + optional: true + "@react-native/metro-config": + optional: true + checksum: 10c0/23d09b4b7e7324efb563f96e7d6fe672bd6c6e2e2a81e60cb60ef017765e43790a1ed0b56d2960d73780730fe3f13b2a4c53cf88b93a2c44da61bbcd16af8e78 + languageName: node + linkType: hard + "@react-native/debugger-frontend@npm:0.81.5": version: 0.81.5 resolution: "@react-native/debugger-frontend@npm:0.81.5" @@ -5911,6 +5991,24 @@ __metadata: languageName: node linkType: hard +"@react-native/debugger-frontend@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/debugger-frontend@npm:0.85.3" + checksum: 10c0/4e52fc9f10051d0d1ae225ff5c1d0fbcf5e5cc2fdfa13f5985b8578352a3196bd0ad6c7714f62901496be7f231e593308964a469f26575680f558f48e6161a2e + languageName: node + linkType: hard + +"@react-native/debugger-shell@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/debugger-shell@npm:0.85.3" + dependencies: + cross-spawn: "npm:^7.0.6" + debug: "npm:^4.4.0" + fb-dotslash: "npm:0.5.8" + checksum: 10c0/2782929d1352c323cc33289fb2b08a8d05b4df5b59af7d588958729f59db966ffd6d73e2df86a66b240f005e23f796829b56978f4a0152a6837e67fd0fae1dd0 + languageName: node + linkType: hard + "@react-native/dev-middleware@npm:0.81.5": version: 0.81.5 resolution: "@react-native/dev-middleware@npm:0.81.5" @@ -5930,6 +6028,26 @@ __metadata: languageName: node linkType: hard +"@react-native/dev-middleware@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/dev-middleware@npm:0.85.3" + dependencies: + "@isaacs/ttlcache": "npm:^1.4.1" + "@react-native/debugger-frontend": "npm:0.85.3" + "@react-native/debugger-shell": "npm:0.85.3" + chrome-launcher: "npm:^0.15.2" + chromium-edge-launcher: "npm:^0.3.0" + connect: "npm:^3.6.5" + debug: "npm:^4.4.0" + invariant: "npm:^2.2.4" + nullthrows: "npm:^1.1.1" + open: "npm:^7.0.3" + serve-static: "npm:^1.16.2" + ws: "npm:^7.5.10" + checksum: 10c0/6adb5e0ecf933f1936eaf993bc39dd8afd4b20e1f1f6525f2c1428fb9000855a764fa93a4043a1bb1678ec827d9c5fca2d6082a928c3dac49f51ea882d991011 + languageName: node + linkType: hard + "@react-native/gradle-plugin@npm:0.81.5": version: 0.81.5 resolution: "@react-native/gradle-plugin@npm:0.81.5" @@ -5937,6 +6055,13 @@ __metadata: languageName: node linkType: hard +"@react-native/gradle-plugin@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/gradle-plugin@npm:0.85.3" + checksum: 10c0/a5a57a0f0783cc31f3daec9e092c32774e72dbae334e9a3eaa86a4ff55f6563069003213d72efc2666e2381b73c653a2f8a4af42c8efd5a974eb848233984a2c + languageName: node + linkType: hard + "@react-native/js-polyfills@npm:0.81.5": version: 0.81.5 resolution: "@react-native/js-polyfills@npm:0.81.5" @@ -5944,6 +6069,13 @@ __metadata: languageName: node linkType: hard +"@react-native/js-polyfills@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/js-polyfills@npm:0.85.3" + checksum: 10c0/67c4ed8234cbeb6d5250b06901f352ddd5348f9aad88aac8a06dc884f42e0da22ff2e1ca8e06c7d03e1e659486bdb9d7682f34858ebf9398d7ae199ce02b3749 + languageName: node + linkType: hard + "@react-native/normalize-colors@npm:0.81.5": version: 0.81.5 resolution: "@react-native/normalize-colors@npm:0.81.5" @@ -5951,6 +6083,13 @@ __metadata: languageName: node linkType: hard +"@react-native/normalize-colors@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/normalize-colors@npm:0.85.3" + checksum: 10c0/343cdbe79e51b0f62e13fcf8620d9c72d7d956d18e9b1b04380a231cff1ec43ff2c54f1ed7a9b975bc23ed4879b96520a7c88aa1c8c7f17a99c32ef8073cf341 + languageName: node + linkType: hard + "@react-native/normalize-colors@npm:^0.74.1": version: 0.74.89 resolution: "@react-native/normalize-colors@npm:0.74.89" @@ -5975,6 +6114,23 @@ __metadata: languageName: node linkType: hard +"@react-native/virtualized-lists@npm:0.85.3": + version: 0.85.3 + resolution: "@react-native/virtualized-lists@npm:0.85.3" + dependencies: + invariant: "npm:^2.2.4" + nullthrows: "npm:^1.1.1" + peerDependencies: + "@types/react": ^19.2.0 + react: "*" + react-native: 0.85.3 + peerDependenciesMeta: + "@types/react": + optional: true + checksum: 10c0/ff3425a27c90f3d51292533c330338cfb285d1a230129abed315e7ea07be55e62ab395f7b167c0fe5b20385f67f587a8ea17440e6ab18661846cc8a1936f8fdb + languageName: node + linkType: hard + "@react-navigation/bottom-tabs@npm:^7.4.0": version: 7.18.3 resolution: "@react-navigation/bottom-tabs@npm:7.18.3" @@ -6761,12 +6917,12 @@ __metadata: languageName: node linkType: hard -"@types/react@npm:19.1.17": - version: 19.1.17 - resolution: "@types/react@npm:19.1.17" +"@types/react@npm:19.2.14": + version: 19.2.14 + resolution: "@types/react@npm:19.2.14" dependencies: - csstype: "npm:^3.0.2" - checksum: 10c0/8a8369ea00fc961f0884be4d1da4a039b2b6445de9c8b690ed0ebe15acfb0b1f27005278fef1fe39a1722a30f4415778b790d0089e2b30019371c61355ea316f + csstype: "npm:^3.2.2" + checksum: 10c0/7d25bf41b57719452d86d2ac0570b659210402707313a36ee612666bf11275a1c69824f8c3ee1fdca077ccfe15452f6da8f1224529b917050eb2d861e52b59b7 languageName: node linkType: hard @@ -7822,6 +7978,13 @@ __metadata: languageName: node linkType: hard +"@webgpu/types@npm:0.1.65": + version: 0.1.65 + resolution: "@webgpu/types@npm:0.1.65" + checksum: 10c0/cc9de41ae18432f292b5e41b1ef1198d200317fd3af424893a75b861fdcff272083e7cb2d51ba79b4deee4d1e93ecec8063d9e6f87d57c69217b48505971c858 + languageName: node + linkType: hard + "@xmldom/xmldom@npm:^0.8.8": version: 0.8.11 resolution: "@xmldom/xmldom@npm:0.8.11" @@ -7862,6 +8025,16 @@ __metadata: languageName: node linkType: hard +"accepts@npm:^2.0.0": + version: 2.0.0 + resolution: "accepts@npm:2.0.0" + dependencies: + mime-types: "npm:^3.0.0" + negotiator: "npm:^1.0.0" + checksum: 10c0/98374742097e140891546076215f90c32644feacf652db48412329de4c2a529178a81aa500fbb13dd3e6cbf6e68d829037b123ac037fc9a08bcec4b87b358eef + languageName: node + linkType: hard + "acorn-globals@npm:^7.0.0": version: 7.0.1 resolution: "acorn-globals@npm:7.0.1" @@ -8530,6 +8703,15 @@ __metadata: languageName: node linkType: hard +"babel-plugin-syntax-hermes-parser@npm:0.33.3": + version: 0.33.3 + resolution: "babel-plugin-syntax-hermes-parser@npm:0.33.3" + dependencies: + hermes-parser: "npm:0.33.3" + checksum: 10c0/61d9f0014b249247e6d5809b638cec4770769a077d3509b8ad575f62c814b28bdd78157dfddf94b040696497c3b78e69cc14793b0b5c15f893c11dc225cc0e3e + languageName: node + linkType: hard + "babel-plugin-syntax-hermes-parser@npm:^0.32.0": version: 0.32.1 resolution: "babel-plugin-syntax-hermes-parser@npm:0.32.1" @@ -9303,6 +9485,19 @@ __metadata: languageName: node linkType: hard +"chromium-edge-launcher@npm:^0.3.0": + version: 0.3.0 + resolution: "chromium-edge-launcher@npm:0.3.0" + dependencies: + "@types/node": "npm:*" + escape-string-regexp: "npm:^4.0.0" + is-wsl: "npm:^2.2.0" + lighthouse-logger: "npm:^1.0.0" + mkdirp: "npm:^1.0.4" + checksum: 10c0/ad04a75bf53ebed0b7adc5bd133587369b0c2e55c92fe460eb6ccec5efe03c161a7466756173969867a2acbe02dd40449186bd74671dd892520492283d4ff43d + languageName: node + linkType: hard + "ci-info@npm:^2.0.0": version: 2.0.0 resolution: "ci-info@npm:2.0.0" @@ -9890,10 +10085,10 @@ __metadata: languageName: node linkType: hard -"csstype@npm:^3.0.2": - version: 3.1.3 - resolution: "csstype@npm:3.1.3" - checksum: 10c0/80c089d6f7e0c5b2bd83cf0539ab41474198579584fa10d86d0cafe0642202343cbc119e076a0b1aece191989477081415d66c9fefbf3c957fc2fc4b7009f248 +"csstype@npm:^3.2.2": + version: 3.2.3 + resolution: "csstype@npm:3.2.3" + checksum: 10c0/cd29c51e70fa822f1cecd8641a1445bed7063697469d35633b516e60fe8c1bde04b08f6c5b6022136bb669b64c63d4173af54864510fbb4ee23281801841a3ce languageName: node linkType: hard @@ -11977,6 +12172,15 @@ __metadata: languageName: node linkType: hard +"fb-dotslash@npm:0.5.8": + version: 0.5.8 + resolution: "fb-dotslash@npm:0.5.8" + bin: + dotslash: bin/dotslash + checksum: 10c0/6c693ecb8e61cd8571e0ad6a923e0582cf8e481695e906e17c8e31620402e06f8b80d95111a420d2f62349d9bebc2b820bae14c2c54a814e72abdc710dc1d3ed + languageName: node + linkType: hard + "fb-watchman@npm:^2.0.0": version: 2.0.2 resolution: "fb-watchman@npm:2.0.2" @@ -12752,6 +12956,13 @@ __metadata: languageName: node linkType: hard +"hermes-compiler@npm:250829098.0.10": + version: 250829098.0.10 + resolution: "hermes-compiler@npm:250829098.0.10" + checksum: 10c0/ccf02f842dc0257deb45cf508dd9183b163fbb1db3b37aca25943cc4667193722dece99c7fba94d89666560b74210873ab139d741def1863bd440ff515113b27 + languageName: node + linkType: hard + "hermes-estree@npm:0.29.1": version: 0.29.1 resolution: "hermes-estree@npm:0.29.1" @@ -12773,6 +12984,20 @@ __metadata: languageName: node linkType: hard +"hermes-estree@npm:0.33.3": + version: 0.33.3 + resolution: "hermes-estree@npm:0.33.3" + checksum: 10c0/4e04e767a706a93c59d64ef3f114075aeb93b08433655d4f11d310f0785c2a74d5b5041b80bc34d22630dece54865dd93a53fde160d48b8369cfef10dbd0520b + languageName: node + linkType: hard + +"hermes-estree@npm:0.35.0": + version: 0.35.0 + resolution: "hermes-estree@npm:0.35.0" + checksum: 10c0/a88c9dc63b8b3679b1aeb43e72e977597096c1bd7d59978c952f1d6df6d1a517c4a817c70b1b701854996b485adfa66c2fc7f80871029a7f0c04306f6717b59a + languageName: node + linkType: hard + "hermes-parser@npm:0.29.1, hermes-parser@npm:^0.29.1": version: 0.29.1 resolution: "hermes-parser@npm:0.29.1" @@ -12800,6 +13025,24 @@ __metadata: languageName: node linkType: hard +"hermes-parser@npm:0.33.3": + version: 0.33.3 + resolution: "hermes-parser@npm:0.33.3" + dependencies: + hermes-estree: "npm:0.33.3" + checksum: 10c0/f7d69de54c77321d8481e37a323bbac01d180ec982275ef8925ceaaf7e501fc3062593e84cf5da50852f36daffb34d0f5d6cbbef079fd0125a7b91c1fe84f225 + languageName: node + linkType: hard + +"hermes-parser@npm:0.35.0": + version: 0.35.0 + resolution: "hermes-parser@npm:0.35.0" + dependencies: + hermes-estree: "npm:0.35.0" + checksum: 10c0/49d98093a2094758db5b536627c6cf5146b140f66e63143acf471c62f1d3fd8bd6ae10a33f2372f72e3653deda5d4615c6dae89d01248849440916209901fc4a + languageName: node + linkType: hard + "hoist-non-react-statics@npm:^3.3.0": version: 3.3.2 resolution: "hoist-non-react-statics@npm:3.3.2" @@ -14698,6 +14941,19 @@ __metadata: languageName: node linkType: hard +"metro-babel-transformer@npm:0.84.4": + version: 0.84.4 + resolution: "metro-babel-transformer@npm:0.84.4" + dependencies: + "@babel/core": "npm:^7.25.2" + flow-enums-runtime: "npm:^0.0.6" + hermes-parser: "npm:0.35.0" + metro-cache-key: "npm:0.84.4" + nullthrows: "npm:^1.1.1" + checksum: 10c0/d1ac996666334bc1cfe9d399cbf4cd747b675f6f8f758c2317eebcc52bd76046ed864ddb7b270efeb8cf337940a61fb03912e5c859b7cbc54687c2f5c41a9d2a + languageName: node + linkType: hard + "metro-cache-key@npm:0.83.3": version: 0.83.3 resolution: "metro-cache-key@npm:0.83.3" @@ -14707,6 +14963,15 @@ __metadata: languageName: node linkType: hard +"metro-cache-key@npm:0.84.4": + version: 0.84.4 + resolution: "metro-cache-key@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + checksum: 10c0/a82ab6367f11886d960cc8fa1f3aa54f6529fe30c16059c141c3e789084c50838fdd7e1a5528534cd9c11a74c63aa5c6a7461dbfa50e8c449b6141eaf2fd05e0 + languageName: node + linkType: hard + "metro-cache@npm:0.83.3": version: 0.83.3 resolution: "metro-cache@npm:0.83.3" @@ -14719,6 +14984,18 @@ __metadata: languageName: node linkType: hard +"metro-cache@npm:0.84.4": + version: 0.84.4 + resolution: "metro-cache@npm:0.84.4" + dependencies: + exponential-backoff: "npm:^3.1.1" + flow-enums-runtime: "npm:^0.0.6" + https-proxy-agent: "npm:^7.0.5" + metro-core: "npm:0.84.4" + checksum: 10c0/3bf7f3a1f85b4f1af05f4b2c71c78e56fd3262d967ee43f02e9ff6820254063af33a70b6549e3dc5e993a6a0b9df92e9279632ad9a8b1cde2577342f93df45eb + languageName: node + linkType: hard + "metro-config@npm:0.83.3, metro-config@npm:^0.83.1": version: 0.83.3 resolution: "metro-config@npm:0.83.3" @@ -14735,6 +15012,22 @@ __metadata: languageName: node linkType: hard +"metro-config@npm:0.84.4, metro-config@npm:^0.84.3": + version: 0.84.4 + resolution: "metro-config@npm:0.84.4" + dependencies: + connect: "npm:^3.6.5" + flow-enums-runtime: "npm:^0.0.6" + jest-validate: "npm:^29.7.0" + metro: "npm:0.84.4" + metro-cache: "npm:0.84.4" + metro-core: "npm:0.84.4" + metro-runtime: "npm:0.84.4" + yaml: "npm:^2.6.1" + checksum: 10c0/f8aaf7d8cff9b486353b62f4746b0a70f99749bd4061f5ae847524aaedcd9c5a34bf176cbbe12fb33e771e8ed3c1496654b2578fa5ba8b9e4f856f0589744d98 + languageName: node + linkType: hard + "metro-core@npm:0.83.3, metro-core@npm:^0.83.1": version: 0.83.3 resolution: "metro-core@npm:0.83.3" @@ -14746,6 +15039,17 @@ __metadata: languageName: node linkType: hard +"metro-core@npm:0.84.4, metro-core@npm:^0.84.3": + version: 0.84.4 + resolution: "metro-core@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + lodash.throttle: "npm:^4.1.1" + metro-resolver: "npm:0.84.4" + checksum: 10c0/19d859de16b5e082c9c31bed981c579a4e6d31a626c7829b725df9ae0ffb755d0ef7809ba9f8adf22d3921f5ffdd931ed77b21b95ca2ea17895f0c99b3cab831 + languageName: node + linkType: hard + "metro-file-map@npm:0.83.3": version: 0.83.3 resolution: "metro-file-map@npm:0.83.3" @@ -14763,6 +15067,23 @@ __metadata: languageName: node linkType: hard +"metro-file-map@npm:0.84.4": + version: 0.84.4 + resolution: "metro-file-map@npm:0.84.4" + dependencies: + debug: "npm:^4.4.0" + fb-watchman: "npm:^2.0.0" + flow-enums-runtime: "npm:^0.0.6" + graceful-fs: "npm:^4.2.4" + invariant: "npm:^2.2.4" + jest-worker: "npm:^29.7.0" + micromatch: "npm:^4.0.4" + nullthrows: "npm:^1.1.1" + walker: "npm:^1.0.7" + checksum: 10c0/09ca829570d1d6dc5beb0534da8a7f2bfcae5415b0974fd5f58b4a05da95dbafdd47f7dc8dedeb11b6562ee9a92c4d918466d02a05cda6e1eaf2c400cbbe6fb4 + languageName: node + linkType: hard + "metro-minify-terser@npm:0.83.3": version: 0.83.3 resolution: "metro-minify-terser@npm:0.83.3" @@ -14773,6 +15094,16 @@ __metadata: languageName: node linkType: hard +"metro-minify-terser@npm:0.84.4": + version: 0.84.4 + resolution: "metro-minify-terser@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + terser: "npm:^5.15.0" + checksum: 10c0/c9b36c2adb8254c38bdedad9da8bf2b7fae7f45cbd883e590430a5fc9cad808af24dd08a9420925e15733dab886528ad553e3eeb3faffc53d3ad80e7e03e5f6d + languageName: node + linkType: hard + "metro-resolver@npm:0.83.3": version: 0.83.3 resolution: "metro-resolver@npm:0.83.3" @@ -14782,6 +15113,15 @@ __metadata: languageName: node linkType: hard +"metro-resolver@npm:0.84.4": + version: 0.84.4 + resolution: "metro-resolver@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + checksum: 10c0/468334270598222e15cbee32af51a3b5e1f4fa6869794955b95d1134b28a58594e8e3879e841ccf00bbb5cd86c689a4481714d6c6a464931987d5333d2c55f80 + languageName: node + linkType: hard + "metro-runtime@npm:0.83.3, metro-runtime@npm:^0.83.1": version: 0.83.3 resolution: "metro-runtime@npm:0.83.3" @@ -14792,6 +15132,16 @@ __metadata: languageName: node linkType: hard +"metro-runtime@npm:0.84.4, metro-runtime@npm:^0.84.3": + version: 0.84.4 + resolution: "metro-runtime@npm:0.84.4" + dependencies: + "@babel/runtime": "npm:^7.25.0" + flow-enums-runtime: "npm:^0.0.6" + checksum: 10c0/e2b2e819027940c6bbd081e5650238d52b6c6d78561cd486b8c10cd1e7fce0213c66fa7f885e37ad5377fcd5726b1c9e473fba6de13938cdf2c966e82968c05f + languageName: node + linkType: hard + "metro-source-map@npm:0.83.3, metro-source-map@npm:^0.83.1": version: 0.83.3 resolution: "metro-source-map@npm:0.83.3" @@ -14810,6 +15160,23 @@ __metadata: languageName: node linkType: hard +"metro-source-map@npm:0.84.4, metro-source-map@npm:^0.84.3": + version: 0.84.4 + resolution: "metro-source-map@npm:0.84.4" + dependencies: + "@babel/traverse": "npm:^7.29.0" + "@babel/types": "npm:^7.29.0" + flow-enums-runtime: "npm:^0.0.6" + invariant: "npm:^2.2.4" + metro-symbolicate: "npm:0.84.4" + nullthrows: "npm:^1.1.1" + ob1: "npm:0.84.4" + source-map: "npm:^0.5.6" + vlq: "npm:^1.0.0" + checksum: 10c0/39df4524022e07aa4b4d09dd874a9509eb9e2e1e491e80a35099020347ab6be2407851b026452296aad314b0eb7ecf14f9b6bab96bd7c31d47d8b1eb30279aaf + languageName: node + linkType: hard + "metro-symbolicate@npm:0.83.3": version: 0.83.3 resolution: "metro-symbolicate@npm:0.83.3" @@ -14826,6 +15193,22 @@ __metadata: languageName: node linkType: hard +"metro-symbolicate@npm:0.84.4": + version: 0.84.4 + resolution: "metro-symbolicate@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + invariant: "npm:^2.2.4" + metro-source-map: "npm:0.84.4" + nullthrows: "npm:^1.1.1" + source-map: "npm:^0.5.6" + vlq: "npm:^1.0.0" + bin: + metro-symbolicate: src/index.js + checksum: 10c0/416a9ef694150a8ec708187743b74ab67e0b4fec39c64610b3771b584830117670a62acb9aa824f84a44efbb1cfec07aaf943d1aaf349d977eecf7c72bd8c0bf + languageName: node + linkType: hard + "metro-transform-plugins@npm:0.83.3": version: 0.83.3 resolution: "metro-transform-plugins@npm:0.83.3" @@ -14840,6 +15223,20 @@ __metadata: languageName: node linkType: hard +"metro-transform-plugins@npm:0.84.4": + version: 0.84.4 + resolution: "metro-transform-plugins@npm:0.84.4" + dependencies: + "@babel/core": "npm:^7.25.2" + "@babel/generator": "npm:^7.29.1" + "@babel/template": "npm:^7.28.6" + "@babel/traverse": "npm:^7.29.0" + flow-enums-runtime: "npm:^0.0.6" + nullthrows: "npm:^1.1.1" + checksum: 10c0/7edb0c0d3655e9f5f5fb8bd8221ec297394b8730c959a3245ea81e50da8177ad7782f21696201a0dcb922281efd919e9548d5b819d8338e52d4b130f06333123 + languageName: node + linkType: hard + "metro-transform-worker@npm:0.83.3": version: 0.83.3 resolution: "metro-transform-worker@npm:0.83.3" @@ -14861,6 +15258,27 @@ __metadata: languageName: node linkType: hard +"metro-transform-worker@npm:0.84.4": + version: 0.84.4 + resolution: "metro-transform-worker@npm:0.84.4" + dependencies: + "@babel/core": "npm:^7.25.2" + "@babel/generator": "npm:^7.29.1" + "@babel/parser": "npm:^7.29.0" + "@babel/types": "npm:^7.29.0" + flow-enums-runtime: "npm:^0.0.6" + metro: "npm:0.84.4" + metro-babel-transformer: "npm:0.84.4" + metro-cache: "npm:0.84.4" + metro-cache-key: "npm:0.84.4" + metro-minify-terser: "npm:0.84.4" + metro-source-map: "npm:0.84.4" + metro-transform-plugins: "npm:0.84.4" + nullthrows: "npm:^1.1.1" + checksum: 10c0/95924f9bcaf6df931bba2783f440d8fab29909bdde8cecdcc3bc7603e7de71e51728a34288f045694b616c94216d1fc683493b8a470e074c9c8a7f220aa9f9b5 + languageName: node + linkType: hard + "metro@npm:0.83.3, metro@npm:^0.83.1": version: 0.83.3 resolution: "metro@npm:0.83.3" @@ -14911,6 +15329,55 @@ __metadata: languageName: node linkType: hard +"metro@npm:0.84.4, metro@npm:^0.84.3": + version: 0.84.4 + resolution: "metro@npm:0.84.4" + dependencies: + "@babel/code-frame": "npm:^7.29.0" + "@babel/core": "npm:^7.25.2" + "@babel/generator": "npm:^7.29.1" + "@babel/parser": "npm:^7.29.0" + "@babel/template": "npm:^7.28.6" + "@babel/traverse": "npm:^7.29.0" + "@babel/types": "npm:^7.29.0" + accepts: "npm:^2.0.0" + ci-info: "npm:^2.0.0" + connect: "npm:^3.6.5" + debug: "npm:^4.4.0" + error-stack-parser: "npm:^2.0.6" + flow-enums-runtime: "npm:^0.0.6" + graceful-fs: "npm:^4.2.4" + hermes-parser: "npm:0.35.0" + image-size: "npm:^1.0.2" + invariant: "npm:^2.2.4" + jest-worker: "npm:^29.7.0" + jsc-safe-url: "npm:^0.2.2" + lodash.throttle: "npm:^4.1.1" + metro-babel-transformer: "npm:0.84.4" + metro-cache: "npm:0.84.4" + metro-cache-key: "npm:0.84.4" + metro-config: "npm:0.84.4" + metro-core: "npm:0.84.4" + metro-file-map: "npm:0.84.4" + metro-resolver: "npm:0.84.4" + metro-runtime: "npm:0.84.4" + metro-source-map: "npm:0.84.4" + metro-symbolicate: "npm:0.84.4" + metro-transform-plugins: "npm:0.84.4" + metro-transform-worker: "npm:0.84.4" + mime-types: "npm:^3.0.1" + nullthrows: "npm:^1.1.1" + serialize-error: "npm:^2.1.0" + source-map: "npm:^0.5.6" + throat: "npm:^5.0.0" + ws: "npm:^7.5.10" + yargs: "npm:^17.6.2" + bin: + metro: src/cli.js + checksum: 10c0/ff92915119db29cd855274f3789d391cba83c50cb92e22d1e9b8c729e7f6d39495e32540a22ca4c6591eea6a847ade49fcfa5faab01b2300227e3f1fc7df359c + languageName: node + linkType: hard + "micromatch@npm:^4.0.4, micromatch@npm:^4.0.8": version: 4.0.8 resolution: "micromatch@npm:4.0.8" @@ -14928,7 +15395,7 @@ __metadata: languageName: node linkType: hard -"mime-db@npm:>= 1.43.0 < 2": +"mime-db@npm:>= 1.43.0 < 2, mime-db@npm:^1.54.0": version: 1.54.0 resolution: "mime-db@npm:1.54.0" checksum: 10c0/8d907917bc2a90fa2df842cdf5dfeaf509adc15fe0531e07bb2f6ab15992416479015828d6a74200041c492e42cce3ebf78e5ce714388a0a538ea9c53eece284 @@ -14944,6 +15411,15 @@ __metadata: languageName: node linkType: hard +"mime-types@npm:^3.0.0, mime-types@npm:^3.0.1": + version: 3.0.2 + resolution: "mime-types@npm:3.0.2" + dependencies: + mime-db: "npm:^1.54.0" + checksum: 10c0/35a0dd1035d14d185664f346efcdb72e93ef7a9b6e9ae808bd1f6358227010267fab52657b37562c80fc888ff76becb2b2938deb5e730818b7983bf8bd359767 + languageName: node + linkType: hard + "mime@npm:1.6.0": version: 1.6.0 resolution: "mime@npm:1.6.0" @@ -15456,6 +15932,15 @@ __metadata: languageName: node linkType: hard +"ob1@npm:0.84.4": + version: 0.84.4 + resolution: "ob1@npm:0.84.4" + dependencies: + flow-enums-runtime: "npm:^0.0.6" + checksum: 10c0/8bf3a3bdc2b27f1b1b60569c31ff2d9d829025f9a1ce7388b5e810242e48672c8d6b24e5972d6e30aef4d84f6894d12b13d0c6c418460d031da1972b96920bba + languageName: node + linkType: hard + "object-assign@npm:^4.0.1, object-assign@npm:^4.1.0, object-assign@npm:^4.1.1": version: 4.1.1 resolution: "object-assign@npm:4.1.1" @@ -16543,6 +17028,16 @@ __metadata: languageName: node linkType: hard +"react-native-nitro-modules@npm:0.35.7": + version: 0.35.7 + resolution: "react-native-nitro-modules@npm:0.35.7" + peerDependencies: + react: "*" + react-native: "*" + checksum: 10c0/f0c9167655032952bfdadd6802eade60796df2021e29a451ec205843dd6e95c6cb274d77c77534703a5d5d55e45c6f7ca1900a94813de3e0bdb0f3aa654cf3fc + languageName: node + linkType: hard + "react-native-reanimated@npm:~4.1.1": version: 4.1.7 resolution: "react-native-reanimated@npm:4.1.7" @@ -16592,6 +17087,31 @@ __metadata: languageName: node linkType: hard +"react-native-vision-camera-worklets@npm:5.0.10": + version: 5.0.10 + resolution: "react-native-vision-camera-worklets@npm:5.0.10" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-modules: "*" + react-native-vision-camera: "*" + react-native-worklets: "*" + checksum: 10c0/fb93bb51c564a65517d6642f3d7a22714bba1906b6d60fa0a5977ba1828dfb2990dba61193d4df1ced03313044d8b08f285934282e8fca8d2450d341c7a6da1e + languageName: node + linkType: hard + +"react-native-vision-camera@npm:5.0.10": + version: 5.0.10 + resolution: "react-native-vision-camera@npm:5.0.10" + peerDependencies: + react: "*" + react-native: "*" + react-native-nitro-image: "*" + react-native-nitro-modules: "*" + checksum: 10c0/f51164b304bc0f2f72440be3097de6228c8f3c34144a74daa5c813cddba89ca945d8df9cb615fc67da5f59b020a89497de3f78ba427650fa2e7e5170ed6f0fa9 + languageName: node + linkType: hard + "react-native-web@npm:~0.21.0": version: 0.21.2 resolution: "react-native-web@npm:0.21.2" @@ -16611,6 +17131,23 @@ __metadata: languageName: node linkType: hard +"react-native-webgpu@npm:0.5.15": + version: 0.5.15 + resolution: "react-native-webgpu@npm:0.5.15" + peerDependencies: + react: "*" + react-native: ">=0.81.0" + react-native-reanimated: ">=4.2.1" + react-native-worklets: ">=0.7.2" + peerDependenciesMeta: + react-native-reanimated: + optional: true + react-native-worklets: + optional: true + checksum: 10c0/91430eb36a5ff5f7e43519a79b384bc9648776c2b4788900e8f9154a706cc8c60c101decc3ef32631cef300f2f636e9aeb038ccf90b15dd8cbb7f8381f39fa4b + languageName: node + linkType: hard + "react-native-worklets@npm:0.5.1": version: 0.5.1 resolution: "react-native-worklets@npm:0.5.1" @@ -16634,6 +17171,31 @@ __metadata: languageName: node linkType: hard +"react-native-worklets@npm:>=0.9.0": + version: 0.10.2 + resolution: "react-native-worklets@npm:0.10.2" + dependencies: + "@babel/plugin-transform-arrow-functions": "npm:^7.27.1" + "@babel/plugin-transform-class-properties": "npm:^7.28.6" + "@babel/plugin-transform-classes": "npm:^7.28.6" + "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.28.6" + "@babel/plugin-transform-optional-chaining": "npm:^7.28.6" + "@babel/plugin-transform-shorthand-properties": "npm:^7.27.1" + "@babel/plugin-transform-template-literals": "npm:^7.27.1" + "@babel/plugin-transform-unicode-regex": "npm:^7.27.1" + "@babel/preset-typescript": "npm:^7.28.5" + "@babel/types": "npm:^7.27.1" + convert-source-map: "npm:^2.0.0" + semver: "npm:^7.7.4" + peerDependencies: + "@babel/core": "*" + "@react-native/metro-config": "*" + react: "*" + react-native: 0.83 - 0.86 + checksum: 10c0/5c4aa9940fb63cd1b0585b618fff84716c028011c9235e10d451b6436c7fb4dec9474a5dd9fa38d74aedd4a5b46f916470fb73aedaca4b0ba47b364d26fbf193 + languageName: node + linkType: hard + "react-native@npm:0.81.5": version: 0.81.5 resolution: "react-native@npm:0.81.5" @@ -16684,6 +17246,57 @@ __metadata: languageName: node linkType: hard +"react-native@npm:0.85.3": + version: 0.85.3 + resolution: "react-native@npm:0.85.3" + dependencies: + "@react-native/assets-registry": "npm:0.85.3" + "@react-native/codegen": "npm:0.85.3" + "@react-native/community-cli-plugin": "npm:0.85.3" + "@react-native/gradle-plugin": "npm:0.85.3" + "@react-native/js-polyfills": "npm:0.85.3" + "@react-native/normalize-colors": "npm:0.85.3" + "@react-native/virtualized-lists": "npm:0.85.3" + abort-controller: "npm:^3.0.0" + anser: "npm:^1.4.9" + ansi-regex: "npm:^5.0.0" + babel-plugin-syntax-hermes-parser: "npm:0.33.3" + base64-js: "npm:^1.5.1" + commander: "npm:^12.0.0" + flow-enums-runtime: "npm:^0.0.6" + hermes-compiler: "npm:250829098.0.10" + invariant: "npm:^2.2.4" + memoize-one: "npm:^5.0.0" + metro-runtime: "npm:^0.84.3" + metro-source-map: "npm:^0.84.3" + nullthrows: "npm:^1.1.1" + pretty-format: "npm:^29.7.0" + promise: "npm:^8.3.0" + react-devtools-core: "npm:^6.1.5" + react-refresh: "npm:^0.14.0" + regenerator-runtime: "npm:^0.13.2" + scheduler: "npm:0.27.0" + semver: "npm:^7.1.3" + stacktrace-parser: "npm:^0.1.10" + tinyglobby: "npm:^0.2.15" + whatwg-fetch: "npm:^3.0.0" + ws: "npm:^7.5.10" + yargs: "npm:^17.6.2" + peerDependencies: + "@react-native/jest-preset": 0.85.3 + "@types/react": ^19.1.1 + react: ^19.2.3 + peerDependenciesMeta: + "@react-native/jest-preset": + optional: true + "@types/react": + optional: true + bin: + react-native: cli.js + checksum: 10c0/365bd8ea2de1a707b753e249f17d9f18d993fd80edf76d4316e133ab6d10d51b9ed6912b952fa2173f56a3b8fbef34f179b686907e7d7c06b6e605fbf1dce895 + languageName: node + linkType: hard + "react-refresh@npm:^0.14.0, react-refresh@npm:^0.14.2": version: 0.14.2 resolution: "react-refresh@npm:0.14.2" @@ -16768,6 +17381,13 @@ __metadata: languageName: node linkType: hard +"react@npm:19.2.3": + version: 19.2.3 + resolution: "react@npm:19.2.3" + checksum: 10c0/094220b3ba3a76c1b668f972ace1dd15509b157aead1b40391d1c8e657e720c201d9719537375eff08f5e0514748c0319063392a6f000e31303aafc4471f1436 + languageName: node + linkType: hard + "readable-stream@npm:^2.0.5": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" @@ -17369,7 +17989,7 @@ __metadata: languageName: node linkType: hard -"scheduler@npm:^0.27.0": +"scheduler@npm:0.27.0, scheduler@npm:^0.27.0": version: 0.27.0 resolution: "scheduler@npm:0.27.0" checksum: 10c0/4f03048cb05a3c8fddc45813052251eca00688f413a3cee236d984a161da28db28ba71bd11e7a3dd02f7af84ab28d39fb311431d3b3772fed557945beb00c452 @@ -17421,7 +18041,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.7.2": +"semver@npm:^7.7.2, semver@npm:^7.7.4": version: 7.8.5 resolution: "semver@npm:7.8.5" bin: From 1600a11c00afad474dad50feb809c977c6d6186b Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 10:35:32 +0200 Subject: [PATCH 02/19] bump react-native-webrtc --- .husky/pre-commit | 0 packages/react-native-webrtc | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 .husky/pre-commit diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100644 new mode 100755 diff --git a/packages/react-native-webrtc b/packages/react-native-webrtc index 140df716..86d2a0dc 160000 --- a/packages/react-native-webrtc +++ b/packages/react-native-webrtc @@ -1 +1 @@ -Subproject commit 140df716ec61070d5891d74decc6df4c91ce3a14 +Subproject commit 86d2a0dce24bd42dea2bbe178e918a9d193c2501 From cc844d6448ad8b909df54115b30e01a8bb612e1f Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 12:42:42 +0200 Subject: [PATCH 03/19] Remove custom source track type override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A custom source is custom by definition — publishing its video as a `camera` track was a footgun. Drop UseCustomSourceOptions / CustomSourceTrackTypes and the videoType threading across react-client, mobile-client, and react-native-vision-camera-source. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mobile-client/src/index.ts | 1 - packages/mobile-client/src/overrides/hooks.ts | 5 ++-- packages/mobile-client/src/overrides/types.ts | 8 ------- .../hooks/internal/useCustomSourceManager.ts | 14 +++++------ .../react-client/src/hooks/useCustomSource.ts | 23 +++---------------- packages/react-client/src/index.ts | 2 +- packages/react-client/src/types/internal.ts | 13 ----------- .../README.md | 8 +++---- .../src/internal/useManagedCustomSource.ts | 8 ++----- .../src/useVisionCameraSource.ts | 12 +++------- .../src/webgpu/useVisionCameraWebGpuSource.ts | 10 +------- 11 files changed, 21 insertions(+), 83 deletions(-) diff --git a/packages/mobile-client/src/index.ts b/packages/mobile-client/src/index.ts index 56d8f0a9..14654e76 100644 --- a/packages/mobile-client/src/index.ts +++ b/packages/mobile-client/src/index.ts @@ -71,7 +71,6 @@ export type { UseMicrophoneResult, UseScreenShareResult, UseCustomSourceResult, - UseCustomSourceOptions, UseInitializeDevicesReturn, Track, RemoteTrack, diff --git a/packages/mobile-client/src/overrides/hooks.ts b/packages/mobile-client/src/overrides/hooks.ts index d822f888..6ddf72b8 100644 --- a/packages/mobile-client/src/overrides/hooks.ts +++ b/packages/mobile-client/src/overrides/hooks.ts @@ -3,7 +3,6 @@ import { type TracksMiddleware as ReactTracksMiddleware, useCamera as useCameraReactClient, useCustomSource as useCustomSourceReactClient, - type UseCustomSourceOptions, useInitializeDevices as useInitializeDevicesReactClient, useLivestreamStreamer as useLivestreamStreamerReactClient, useLivestreamViewer as useLivestreamViewerReactClient, @@ -84,8 +83,8 @@ export function useScreenShare(): UseScreenShareResult { }; } -export function useCustomSource(sourceId: T, options?: UseCustomSourceOptions) { - const result = useCustomSourceReactClient(sourceId, options); +export function useCustomSource(sourceId: T) { + const result = useCustomSourceReactClient(sourceId); return { ...result, stream: result.stream as RNMediaStream | undefined, diff --git a/packages/mobile-client/src/overrides/types.ts b/packages/mobile-client/src/overrides/types.ts index 2c67b9a3..4fcde84f 100644 --- a/packages/mobile-client/src/overrides/types.ts +++ b/packages/mobile-client/src/overrides/types.ts @@ -7,7 +7,6 @@ import type { Track as ReactClientTrack, useCamera as useCameraReactClient, useCustomSource as useCustomSourceReactClient, - UseCustomSourceOptions as ReactClientUseCustomSourceOptions, useInitializeDevices as useInitializeDevicesReactClient, UseLivestreamStreamerResult as ReactClientUseLivestreamStreamerResult, UseLivestreamViewerResult as ReactClientUseLivestreamViewerResult, @@ -103,13 +102,6 @@ export type UseCustomSourceResult = Omit Promise; }; -/** - * Options controlling how a custom source's tracks are published. - * Pass `videoType: 'camera'` to publish the source's video as a regular camera - * track (a "virtual camera"), so receivers bucket it as the peer's camera track. - */ -export type UseCustomSourceOptions = ReactClientUseCustomSourceOptions; - export type UseInitializeDevicesReturn = { initializeDevices: ( ...args: Parameters['initializeDevices']> diff --git a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts index 7469daa4..d5456f1b 100644 --- a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts +++ b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts @@ -1,7 +1,7 @@ import { type FishjamClient, type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client"; import { useCallback, useEffect, useMemo, useState } from "react"; -import type { CustomSourceState, CustomSourceTracks, CustomSourceTrackTypes } from "../../types/internal"; +import type { CustomSourceState, CustomSourceTracks } from "../../types/internal"; import type { PeerStatus } from "../../types/public"; type CustomSourceManagerProps = { @@ -11,7 +11,7 @@ type CustomSourceManagerProps = { }; export type CustomSourceManager = { - setStream: (sourceId: string, stream: MediaStream | null, trackTypes?: CustomSourceTrackTypes) => Promise; + setStream: (sourceId: string, stream: MediaStream | null) => Promise; getSource: (sourceId: string) => CustomSourceState | undefined; }; @@ -55,14 +55,12 @@ export function useCustomSourceManager({ const promises = []; const displayName = getDisplayName(); - const videoType = source.trackTypes?.videoType ?? "customVideo"; - const audioType = source.trackTypes?.audioType ?? "customAudio"; if (video) { - const videoMetadata = { type: videoType, displayName, paused: false } as const; + const videoMetadata = { type: "customVideo", displayName, paused: false } as const; promises.push(addTrackToFishjamClient(video, videoMetadata)); } if (audio) { - const audioMetadata = { type: audioType, displayName, paused: false } as const; + const audioMetadata = { type: "customAudio", displayName, paused: false } as const; promises.push(addTrackToFishjamClient(audio, audioMetadata)); } @@ -89,14 +87,14 @@ export function useCustomSourceManager({ const getSource = useCallback((sourceId: string) => sources[sourceId], [sources]); const setStream = useCallback( - async (sourceId: string, stream: MediaStream | null, trackTypes?: CustomSourceTrackTypes) => { + async (sourceId: string, stream: MediaStream | null) => { const oldSource = sources[sourceId]; if (stream === oldSource?.stream) return; if (oldSource?.trackIds) await removeTracks(oldSource.trackIds); if (stream !== null) { - setSources((old) => ({ ...old, [sourceId]: { stream, trackTypes } })); + setSources((old) => ({ ...old, [sourceId]: { stream } })); return; } if (!oldSource) return; diff --git a/packages/react-client/src/hooks/useCustomSource.ts b/packages/react-client/src/hooks/useCustomSource.ts index befdebdf..a1b27fb3 100644 --- a/packages/react-client/src/hooks/useCustomSource.ts +++ b/packages/react-client/src/hooks/useCustomSource.ts @@ -1,28 +1,14 @@ import { useCallback, useContext, useMemo } from "react"; import { CustomSourceContext } from "../contexts/customSource"; -import type { CustomSourceTrackTypes } from "../types/internal"; - -/** - * Options controlling how a custom source's tracks are published. - * @group Hooks - */ -export type UseCustomSourceOptions = CustomSourceTrackTypes; /** * This hook can register/deregister a custom MediaStream with Fishjam. * - * By default a custom source's video is published as a `customVideo` track and - * its audio as a `customAudio` track. Pass `options.videoType: 'camera'` to - * publish the video as a regular camera track (a "virtual camera"), so every - * receiver buckets it as the peer's camera track with no receiver-side changes. - * * @param sourceId - Stable id identifying this custom source. - * @param options - Optional track types to publish under. Defaults to - * `{ videoType: 'customVideo', audioType: 'customAudio' }`. * @group Hooks */ -export function useCustomSource(sourceId: T, options?: UseCustomSourceOptions) { +export function useCustomSource(sourceId: T) { const customSourceManager = useContext(CustomSourceContext); if (!customSourceManager) throw Error("useCustomSource must be used within FishjamProvider"); @@ -30,12 +16,9 @@ export function useCustomSource(sourceId: T, options?: UseCust const stream = useMemo(() => getSource(sourceId)?.stream, [getSource, sourceId]); - const videoType = options?.videoType; - const audioType = options?.audioType; - const setStream = useCallback( - (newStream: MediaStream | null) => managerSetStream(sourceId, newStream, { videoType, audioType }), - [managerSetStream, sourceId, videoType, audioType], + (newStream: MediaStream | null) => managerSetStream(sourceId, newStream), + [managerSetStream, sourceId], ); return { diff --git a/packages/react-client/src/index.ts b/packages/react-client/src/index.ts index d1fd6510..5f404ec4 100644 --- a/packages/react-client/src/index.ts +++ b/packages/react-client/src/index.ts @@ -9,7 +9,7 @@ export { useInitializeDevices, UseInitializeDevicesParams } from "./hooks/device export { useMicrophone } from "./hooks/devices/useMicrophone"; export { InitializeDevicesSettings } from "./hooks/internal/devices/useMediaDevices"; export { type JoinRoomConfig, useConnection } from "./hooks/useConnection"; -export { useCustomSource, type UseCustomSourceOptions } from "./hooks/useCustomSource"; +export { useCustomSource } from "./hooks/useCustomSource"; export { useDataChannel } from "./hooks/useDataChannel"; export { type ConnectStreamerConfig, diff --git a/packages/react-client/src/types/internal.ts b/packages/react-client/src/types/internal.ts index 31f003eb..f7f3abdb 100644 --- a/packages/react-client/src/types/internal.ts +++ b/packages/react-client/src/types/internal.ts @@ -47,20 +47,7 @@ export type CustomSourceTracks = { audioId?: string; }; -/** - * Track types a custom source may publish under. Defaults to the "custom" - * variants, but a source can opt into publishing its video as a regular - * `camera` track (a virtual camera) so receivers bucket it as a camera track. - */ -export type CustomSourceTrackTypes = { - /** Metadata type for the source's video track. Defaults to `customVideo`. */ - videoType?: "camera" | "customVideo"; - /** Metadata type for the source's audio track. Defaults to `customAudio`. */ - audioType?: "customAudio"; -}; - export type CustomSourceState = { stream: MediaStream; trackIds?: CustomSourceTracks; - trackTypes?: CustomSourceTrackTypes; }; diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md index 4e21fe7f..6d028931 100644 --- a/packages/react-native-vision-camera-source/README.md +++ b/packages/react-native-vision-camera-source/README.md @@ -28,7 +28,7 @@ function CameraPublisher() { const { hasPermission } = useCameraPermission(); const cameraDevice = useCameraDevices().find((device) => device.position === 'front'); - const { frameOutput, stream } = useVisionCameraSource('my-camera', { videoType: 'camera' }); + const { frameOutput, stream } = useVisionCameraSource('my-camera'); useVisionCamera({ device: cameraDevice, isActive: hasPermission, outputs: [frameOutput] }); @@ -36,8 +36,7 @@ function CameraPublisher() { } ``` -Frames are handed to Fishjam without copying pixels. `videoType: 'camera'` makes receivers treat -the video as the peer's camera track; omit it to publish a separate custom video track. +Frames are handed to Fishjam without copying pixels. ## Publish + run inference @@ -51,7 +50,7 @@ const onFrame = useCallback( [detectPose], ); -const { frameOutput } = useVisionCameraSource('my-camera', { videoType: 'camera', onFrame }); +const { frameOutput } = useVisionCameraSource('my-camera', { onFrame }); ``` The frame is valid only inside your synchronous callback — the hook releases it afterwards. @@ -111,7 +110,6 @@ const onFrame = useCallback( ); const { frameOutput, stream } = useVisionCameraWebGpuSource('my-camera', { - videoType: 'camera', width: 720, height: 1280, cameraShaderBindings: effect?.cameraBindings, diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts b/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts index ee0f5388..e76cab7d 100644 --- a/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts +++ b/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts @@ -6,12 +6,8 @@ import { useEffect, useRef } from 'react'; * Publishes `stream` under `sourceId` via the SDK's custom-source machinery: publish when the * stream appears, unpublish on cleanup. Must be used under `FishjamProvider`. */ -export function useManagedCustomSource( - sourceId: string, - videoType: 'camera' | 'customVideo' | undefined, - stream: MediaStream | null, -): void { - const { setStream } = useCustomSource(sourceId, videoType != null ? { videoType } : undefined); +export function useManagedCustomSource(sourceId: string, stream: MediaStream | null): void { + const { setStream } = useCustomSource(sourceId); // setStream is not referentially stable across renders; going through a ref keeps the effect's // cleanup from unpublishing with a stale instance. diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts index c2796cac..7d35b95b 100644 --- a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -23,12 +23,6 @@ export interface UseVisionCameraSourceOptions extends Partial( sourceId: SourceId, options: UseVisionCameraSourceOptions = {}, ): UseVisionCameraSourceResult { - const { enabled = true, videoType, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; + const { enabled = true, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; const { track, stream, error } = useManagedForwardTrack(enabled); - useManagedCustomSource(sourceId, videoType, stream); + useManagedCustomSource(sourceId, stream); const timestampState = useMemo(() => createFrameTimestampState(), []); diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index a17c74d6..a9f1e2b8 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -33,12 +33,6 @@ export interface UseVisionCameraWebGpuSourceOptions extends Partial( ): UseVisionCameraWebGpuSourceResult { const { enabled = true, - videoType, width, height, poolSize = DEFAULT_POOL_SIZE, @@ -148,7 +140,7 @@ export function useVisionCameraWebGpuSource( bufferDescriptors, error: trackError, } = useManagedPooledTrack(enabled, width, height, poolSize); - useManagedCustomSource(sourceId, videoType, stream); + useManagedCustomSource(sourceId, stream); const runtime = getWebGpuRuntime(); const outputSurfaceFormat = getOutputSurfaceFormat(); From 7cf0d8baf45b762a918096a98b4b62cfdcb966ce Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 12:54:45 +0200 Subject: [PATCH 04/19] Stabilize useCustomSource setStream, drop useManagedCustomSource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setStream was referentially unstable because its useCallback closed over the sources state, so it changed identity on every publish. Read sources through a ref instead; setStream is now stable. With that fixed, the useManagedCustomSource adapter (which existed only to work around the instability via a ref) is redundant — inline its publish effect into useVisionCameraSource and useVisionCameraWebGpuSource. Add useCustomSourceManager tests pinning setStream stability and the publish/unpublish lifecycle. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../hooks/internal/useCustomSourceManager.ts | 12 ++- .../src/tests/useCustomSourceManager.spec.ts | 86 +++++++++++++++++++ .../src/internal/useManagedCustomSource.ts | 26 ------ .../src/useVisionCameraSource.ts | 15 +++- .../src/webgpu/useVisionCameraWebGpuSource.ts | 15 +++- 5 files changed, 119 insertions(+), 35 deletions(-) create mode 100644 packages/react-client/src/tests/useCustomSourceManager.spec.ts delete mode 100644 packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts diff --git a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts index d5456f1b..2d0b10e0 100644 --- a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts +++ b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts @@ -1,5 +1,5 @@ import { type FishjamClient, type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { CustomSourceState, CustomSourceTracks } from "../../types/internal"; import type { PeerStatus } from "../../types/public"; @@ -21,6 +21,12 @@ export function useCustomSourceManager({ logger, }: CustomSourceManagerProps): CustomSourceManager { const [sources, setSources] = useState>({}); + // setStream reads the current sources synchronously (to diff against the old stream and to + // remove stale tracks), but must stay referentially stable so consumers can safely depend on + // it in effects. Reading through a ref keeps it out of the useCallback deps. + const sourcesRef = useRef(sources); + sourcesRef.current = sources; + const pendingSources = useMemo( () => Object.entries(sources).filter(([_, source]) => source.trackIds === undefined), [sources], @@ -88,7 +94,7 @@ export function useCustomSourceManager({ const setStream = useCallback( async (sourceId: string, stream: MediaStream | null) => { - const oldSource = sources[sourceId]; + const oldSource = sourcesRef.current[sourceId]; if (stream === oldSource?.stream) return; if (oldSource?.trackIds) await removeTracks(oldSource.trackIds); @@ -101,7 +107,7 @@ export function useCustomSourceManager({ setSources((old) => Object.fromEntries(Object.entries(old).filter(([id, _]) => id !== sourceId))); }, - [sources, removeTracks], + [removeTracks], ); useEffect(() => { diff --git a/packages/react-client/src/tests/useCustomSourceManager.spec.ts b/packages/react-client/src/tests/useCustomSourceManager.spec.ts new file mode 100644 index 00000000..5161bb2b --- /dev/null +++ b/packages/react-client/src/tests/useCustomSourceManager.spec.ts @@ -0,0 +1,86 @@ +import type { FishjamClient, Logger } from "@fishjam-cloud/ts-client"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import { FakeMediaStreamTrack } from "fake-mediastreamtrack"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { useCustomSourceManager } from "../hooks/internal/useCustomSourceManager"; +import type { PeerStatus } from "../types/public"; + +function makeStream({ video = true, audio = false } = {}): MediaStream { + const videoTracks = video ? [new FakeMediaStreamTrack({ kind: "video" })] : []; + const audioTracks = audio ? [new FakeMediaStreamTrack({ kind: "audio" })] : []; + return { + getVideoTracks: () => videoTracks, + getAudioTracks: () => audioTracks, + } as unknown as MediaStream; +} + +function makeFishjamClient() { + return { + getLocalPeer: () => ({ metadata: { peer: { displayName: "tester" } } }), + addTrack: vi.fn(async () => "video-id"), + removeTrack: vi.fn(async () => undefined), + on: vi.fn(), + off: vi.fn(), + } as unknown as FishjamClient; +} + +const logger = { warn: vi.fn(), error: vi.fn(), info: vi.fn() } as unknown as Logger; + +describe("useCustomSourceManager", () => { + beforeEach(() => vi.clearAllMocks()); + + it("keeps setStream referentially stable across renders", () => { + const fishjamClient = makeFishjamClient(); + const { result, rerender } = renderHook( + ({ peerStatus }: { peerStatus: PeerStatus }) => useCustomSourceManager({ fishjamClient, peerStatus, logger }), + { initialProps: { peerStatus: "connecting" } }, + ); + + const first = result.current.setStream; + rerender({ peerStatus: "connected" }); + expect(result.current.setStream).toBe(first); + }); + + it("keeps setStream referentially stable after publishing mutates internal state", async () => { + const fishjamClient = makeFishjamClient(); + const { result } = renderHook(() => + // "connecting" so the source stays pending and no async publish effect runs — this isolates + // the referential-stability property from the publish lifecycle. + useCustomSourceManager({ fishjamClient, peerStatus: "connecting", logger }), + ); + + const before = result.current.setStream; + const stream = makeStream(); + await act(async () => { + await result.current.setStream("src-1", stream); + }); + + // Regression guard: setStream used to depend on the `sources` state, so calling it (which + // updates that state) handed back a brand-new function every time. + expect(result.current.setStream).toBe(before); + expect(result.current.getSource("src-1")?.stream).toBe(stream); + }); + + it("publishes a customVideo track when connected and removes it on setStream(null)", async () => { + const fishjamClient = makeFishjamClient(); + const { result } = renderHook(() => useCustomSourceManager({ fishjamClient, peerStatus: "connected", logger })); + + const stream = makeStream(); + await act(async () => { + await result.current.setStream("src-1", stream); + }); + + await waitFor(() => expect(fishjamClient.addTrack).toHaveBeenCalledTimes(1)); + const [publishedTrack, publishedMetadata] = vi.mocked(fishjamClient.addTrack).mock.calls[0]; + expect(publishedTrack).toBe(stream.getVideoTracks()[0]); + expect(publishedMetadata).toEqual({ type: "customVideo", displayName: "tester", paused: false }); + + await act(async () => { + await result.current.setStream("src-1", null); + }); + + expect(fishjamClient.removeTrack).toHaveBeenCalledWith("video-id"); + expect(result.current.getSource("src-1")).toBeUndefined(); + }); +}); diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts b/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts deleted file mode 100644 index e76cab7d..00000000 --- a/packages/react-native-vision-camera-source/src/internal/useManagedCustomSource.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { useCustomSource } from '@fishjam-cloud/react-native-client'; -import type { MediaStream } from '@fishjam-cloud/react-native-webrtc'; -import { useEffect, useRef } from 'react'; - -/** - * Publishes `stream` under `sourceId` via the SDK's custom-source machinery: publish when the - * stream appears, unpublish on cleanup. Must be used under `FishjamProvider`. - */ -export function useManagedCustomSource(sourceId: string, stream: MediaStream | null): void { - const { setStream } = useCustomSource(sourceId); - - // setStream is not referentially stable across renders; going through a ref keeps the effect's - // cleanup from unpublishing with a stale instance. - const setStreamRef = useRef(setStream); - setStreamRef.current = setStream; - - useEffect(() => { - if (stream == null) { - return; - } - void setStreamRef.current(stream); - return () => { - void setStreamRef.current(null); - }; - }, [stream]); -} diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts index 7d35b95b..b5bd4246 100644 --- a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -1,5 +1,6 @@ +import { useCustomSource } from '@fishjam-cloud/react-native-client'; import { forwardFrame, type MediaStream } from '@fishjam-cloud/react-native-webrtc'; -import { useMemo } from 'react'; +import { useEffect, useMemo } from 'react'; import { type CameraFrameOutput, type Frame, @@ -9,7 +10,6 @@ import { } from 'react-native-vision-camera'; import { createFrameTimestampState, normalizeFrameTimestampNanoseconds } from './frameTimestamp'; -import { useManagedCustomSource } from './internal/useManagedCustomSource'; import { useManagedForwardTrack } from './internal/useManagedForwardTrack'; import { rotationDegreesFromOrientation } from './orientation'; @@ -84,7 +84,16 @@ export function useVisionCameraSource( const { enabled = true, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; const { track, stream, error } = useManagedForwardTrack(enabled); - useManagedCustomSource(sourceId, stream); + + // Publish the track's stream under sourceId while it exists, unpublish on cleanup/unmount. + const { setStream } = useCustomSource(sourceId); + useEffect(() => { + if (stream == null) return; + void setStream(stream); + return () => { + void setStream(null); + }; + }, [stream, setStream]); const timestampState = useMemo(() => createFrameTimestampState(), []); diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index a9f1e2b8..d28a8721 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -1,5 +1,6 @@ +import { useCustomSource } from '@fishjam-cloud/react-native-client'; import { type MediaStream, pushFrame } from '@fishjam-cloud/react-native-webrtc'; -import { useMemo } from 'react'; +import { useEffect, useMemo } from 'react'; import { type CameraFrameOutput, type Frame, @@ -10,7 +11,6 @@ import { import { type GPUSharedTextureMemory, GPUTextureUsage } from 'react-native-webgpu'; import { createFrameTimestampState, nextFrameTimestampNanoseconds } from '../frameTimestamp'; -import { useManagedCustomSource } from '../internal/useManagedCustomSource'; import { useManagedPooledTrack } from '../internal/useManagedPooledTrack'; import { rotationDegreesFromOrientation } from '../orientation'; import { type CameraShaderBindings, createCameraBindGroup } from './cameraShaderBindings'; @@ -140,7 +140,16 @@ export function useVisionCameraWebGpuSource( bufferDescriptors, error: trackError, } = useManagedPooledTrack(enabled, width, height, poolSize); - useManagedCustomSource(sourceId, stream); + + // Publish the track's stream under sourceId while it exists, unpublish on cleanup/unmount. + const { setStream } = useCustomSource(sourceId); + useEffect(() => { + if (stream == null) return; + void setStream(stream); + return () => { + void setStream(null); + }; + }, [stream, setStream]); const runtime = getWebGpuRuntime(); const outputSurfaceFormat = getOutputSurfaceFormat(); From 146c62479ca79ba3e5513e1eb8dfde44cec3cd14 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 13:21:12 +0200 Subject: [PATCH 05/19] vision-camera-source: migrate build tsc -> react-native-builder-bob Prep for the TGSL shader port: JS-body TGSL needs the unplugin-typegpu build transform, which tsc can't run. Move the package build to bob with a babel.config.js that runs @babel/preset-typescript + unplugin-typegpu. - module target (configFile:true) so bob defers to babel.config.js; verified the built output preserves 'worklet' directives (for the consumer's react-native-worklets plugin) and resolves TGSL to WGSL. - typescript target emits .d.ts to dist/typescript. - exports/main/types repointed to dist/module + dist/typescript. - add typegpu, unplugin-typegpu, react-native-builder-bob, babel deps. No shader code changed yet; shaders still ship as raw WGSL strings. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../babel.config.js | 16 + .../package.json | 57 +- .../tsconfig.build.json | 9 + yarn.lock | 1119 ++++++++++++++++- 4 files changed, 1168 insertions(+), 33 deletions(-) create mode 100644 packages/react-native-vision-camera-source/babel.config.js create mode 100644 packages/react-native-vision-camera-source/tsconfig.build.json diff --git a/packages/react-native-vision-camera-source/babel.config.js b/packages/react-native-vision-camera-source/babel.config.js new file mode 100644 index 00000000..eeaf9dd7 --- /dev/null +++ b/packages/react-native-vision-camera-source/babel.config.js @@ -0,0 +1,16 @@ +// Build-time Babel config, used only by react-native-builder-bob when compiling this package's +// own source to `dist` (bob is configured with `configFile: true`, so it defers entirely to this +// config instead of its default presets). It is never shipped — the published tarball is `dist` +// only — so it does not affect how consumers' Metro transpiles the built output. +// +// - `@babel/preset-typescript` strips the TypeScript types. It preserves `'worklet'` string +// directives, which must survive into `dist` so the consumer app's react-native-worklets Babel +// plugin can pick them up. We deliberately do NOT run the worklets plugin here — that is the +// consumer's job at app-build time. +// - `unplugin-typegpu/babel` transforms TGSL (`tgpu.fn(... )(js => ...)`) function bodies into the +// tinyest AST that TypeGPU resolves to WGSL at runtime. Without it, TGSL functions throw on +// `tgpu.resolve`. +module.exports = { + presets: [['@babel/preset-typescript', { onlyRemoveTypeImports: true }]], + plugins: [['unplugin-typegpu/babel', { forceTgpuAlias: 'tgpu' }]], +}; diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json index 88f0d29a..f7a42396 100644 --- a/packages/react-native-vision-camera-source/package.json +++ b/packages/react-native-vision-camera-source/package.json @@ -4,21 +4,22 @@ "description": "VisionCamera frame outputs that publish camera frames to Fishjam — zero-copy forwarding and WebGPU-rendered video sources", "license": "Apache-2.0", "author": "Fishjam Team", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "react-native": "dist/index.js", + "main": "./dist/module/index.js", + "module": "./dist/module/index.js", + "types": "./dist/typescript/index.d.ts", + "react-native": "./dist/module/index.js", "exports": { ".": { - "types": "./dist/index.d.ts", - "react-native": "./dist/index.js", - "import": "./dist/index.js", - "default": "./dist/index.js" + "types": "./dist/typescript/index.d.ts", + "react-native": "./dist/module/index.js", + "import": "./dist/module/index.js", + "default": "./dist/module/index.js" }, "./webgpu": { - "types": "./dist/webgpu.d.ts", - "react-native": "./dist/webgpu.js", - "import": "./dist/webgpu.js", - "default": "./dist/webgpu.js" + "types": "./dist/typescript/webgpu.d.ts", + "react-native": "./dist/module/webgpu.js", + "import": "./dist/module/webgpu.js", + "default": "./dist/module/webgpu.js" }, "./package.json": "./package.json" }, @@ -39,11 +40,11 @@ "webgpu" ], "scripts": { - "tsc": "tsc", + "tsc": "tsc --noEmit", "build:webrtc-workspace": "yarn workspace @fishjam-cloud/react-native-webrtc run prepare", - "build": "yarn build:webrtc-workspace && tsc", + "build": "yarn build:webrtc-workspace && bob build", "lint": "eslint . --ext .ts,.tsx --fix", - "prepare": "yarn build:webrtc-workspace && tsc", + "prepare": "yarn build:webrtc-workspace && bob build", "format": "prettier --write . --ignore-path ./.eslintignore", "check:webrtc-published": "node ../../release-automation/check-webrtc-published.mjs" }, @@ -55,6 +56,25 @@ "yarn lint" ] }, + "react-native-builder-bob": { + "source": "src", + "output": "dist", + "targets": [ + [ + "module", + { + "configFile": true + } + ], + [ + "typescript", + { + "project": "tsconfig.build.json", + "tsc": "../../node_modules/.bin/tsc" + } + ] + ] + }, "peerDependencies": { "@fishjam-cloud/react-native-client": "workspace:*", "@fishjam-cloud/react-native-webrtc": "workspace:*", @@ -70,9 +90,12 @@ } }, "dependencies": { - "react-native-worklets": ">=0.9.0" + "react-native-worklets": ">=0.9.0", + "typegpu": "^0.11.8" }, "devDependencies": { + "@babel/core": "^7.29.0", + "@babel/preset-typescript": "^7.27.0", "@fishjam-cloud/react-native-client": "workspace:*", "@fishjam-cloud/react-native-webrtc": "workspace:*", "@types/react": "19.2.14", @@ -82,11 +105,13 @@ "eslint-plugin-prettier": "^5.5.1", "react": "19.2.3", "react-native": "0.85.3", + "react-native-builder-bob": "^0.18.2", "react-native-nitro-modules": "0.35.7", "react-native-vision-camera": "5.0.10", "react-native-vision-camera-worklets": "5.0.10", "react-native-webgpu": "0.5.15", - "typescript": "^5.8.3" + "typescript": "^5.8.3", + "unplugin-typegpu": "^0.11.5" }, "packageManager": "yarn@4.16.0" } diff --git a/packages/react-native-vision-camera-source/tsconfig.build.json b/packages/react-native-vision-camera-source/tsconfig.build.json new file mode 100644 index 00000000..dbe3ee21 --- /dev/null +++ b/packages/react-native-vision-camera-source/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/yarn.lock b/yarn.lock index f6348fb3..8f09c4e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -194,7 +194,7 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.20.0": +"@babel/core@npm:^7.18.5, @babel/core@npm:^7.20.0, @babel/core@npm:^7.29.0": version: 7.29.7 resolution: "@babel/core@npm:7.29.7" dependencies: @@ -654,6 +654,19 @@ __metadata: languageName: node linkType: hard +"@babel/helper-remap-async-to-generator@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-remap-async-to-generator@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": "npm:^7.29.7" + "@babel/helper-wrap-function": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/d71a4f2c7e4523568b1fbeb4d85823bda7ebcd48263b2862ee8194b0a647b612e70d6a64472e0842fc7e557a16e9fa5d3c032af5c1308a342e2a3b49a87d4833 + languageName: node + linkType: hard + "@babel/helper-replace-supers@npm:^7.27.1": version: 7.27.1 resolution: "@babel/helper-replace-supers@npm:7.27.1" @@ -787,6 +800,17 @@ __metadata: languageName: node linkType: hard +"@babel/helper-wrap-function@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-wrap-function@npm:7.29.7" + dependencies: + "@babel/template": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + "@babel/types": "npm:^7.29.7" + checksum: 10c0/d765b341863ede049eb6923b9c20fc79fb6c64995a906b60a70c22fbffb21eef5d5a5cf15948c843047d31e8c902a85fa94ccfe5d30d0631a7b1e40e4d667c70 + languageName: node + linkType: hard + "@babel/helpers@npm:^7.26.10": version: 7.27.0 resolution: "@babel/helpers@npm:7.27.0" @@ -895,6 +919,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-bugfix-firefox-class-in-computed-class-key@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-firefox-class-in-computed-class-key@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/6877a4112797885bcf90f361131e2b63dcbbcb3e334c984c527e1e380eb7e81785219dcf99f1291eef4edc83907cb582204120d97d777807c252f9eb31fd2e6e + languageName: node + linkType: hard + "@babel/plugin-bugfix-safari-class-field-initializer-scope@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-bugfix-safari-class-field-initializer-scope@npm:7.27.1" @@ -906,6 +942,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-bugfix-safari-class-field-initializer-scope@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-safari-class-field-initializer-scope@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/557565fc052b9ab306802b19b9cfc89be9aa7aa709447698dbb5080db356048d4c3dd94ccad8bcb4d5ef3c4002947a340d1c326acde0d95950c3773472f46282 + languageName: node + linkType: hard + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.27.1" @@ -917,6 +964,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/5b949826cfadd9d90c3cfba01cc917f218ba2da05b2a2dc4781bd3805c150eb858ba52d2f3972c8ae6cc887257ac4d114fdda6d695a30510137abae5c76bf054 + languageName: node + linkType: hard + "@babel/plugin-bugfix-safari-rest-destructuring-rhs-array@npm:^7.29.3": version: 7.29.3 resolution: "@babel/plugin-bugfix-safari-rest-destructuring-rhs-array@npm:7.29.3" @@ -929,6 +987,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-bugfix-safari-rest-destructuring-rhs-array@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-safari-rest-destructuring-rhs-array@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-skip-transparent-expression-wrappers": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/727a464410623fb47d47a2803562b8bb9fb4b915de94cc51bd3149937522b85e6a5d19c71207ac5269c51684f282a918785e78e359435d1f4ac889dc4f258855 + languageName: node + linkType: hard + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:7.27.1" @@ -942,6 +1012,19 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-skip-transparent-expression-wrappers": "npm:^7.29.7" + "@babel/plugin-transform-optional-chaining": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.13.0 + checksum: 10c0/9ccc704d1382771b2a6a4f1281b2f63d7be47fb0ebc9890e2f820e2a645b77aaf207ec8e6136854bb059715eac5fd7cab436b054c3b3b4a472b9fd0c73ee98b3 + languageName: node + linkType: hard + "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@npm:^7.28.3": version: 7.28.3 resolution: "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@npm:7.28.3" @@ -966,7 +1049,19 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-proposal-class-properties@npm:^7.12.1": +"@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/547bddf129eed825c0a3c706828c579bfedd0fa850054ded515e90af91fa1a643bb88461e49b59946d95737622766ae0db2c04346ee319e06e49852c270a2c2f + languageName: node + linkType: hard + +"@babel/plugin-proposal-class-properties@npm:^7.12.1, @babel/plugin-proposal-class-properties@npm:^7.17.12": version: 7.18.6 resolution: "@babel/plugin-proposal-class-properties@npm:7.18.6" dependencies: @@ -1099,6 +1194,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-syntax-flow@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-syntax-flow@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/b2e330dd0dc535c7364937ce2b29a06065e60b1d523db75f36ab4fb89e4ba664e2230cbb1937b35712c9a0ebafc3358f323d6e6b22247d5ebad12fdd3e1f6e98 + languageName: node + linkType: hard + "@babel/plugin-syntax-import-assertions@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-syntax-import-assertions@npm:7.27.1" @@ -1121,6 +1227,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-syntax-import-assertions@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-syntax-import-assertions@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/d5628692a0ba2fadf469aaef20f4f0a216259eeb92d31fc23d48c9d201696099691f0dfaa895c657f9c591a7fab94f62545f20196326af1812ebab72cf34e9fe + languageName: node + linkType: hard + "@babel/plugin-syntax-import-attributes@npm:^7.24.7, @babel/plugin-syntax-import-attributes@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-syntax-import-attributes@npm:7.27.1" @@ -1143,6 +1260,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-syntax-import-attributes@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-syntax-import-attributes@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/b9a47e869d8c06676297069895ae34e2bd244ec4c3bdf15002f1e69aed32eef0361044af22a43f271b8de5e23a40534fe6a74a63e7ab98e3d60a74b322444b49 + languageName: node + linkType: hard + "@babel/plugin-syntax-import-meta@npm:^7.10.4": version: 7.10.4 resolution: "@babel/plugin-syntax-import-meta@npm:7.10.4" @@ -1309,7 +1437,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-arrow-functions@npm:^7.0.0-0": +"@babel/plugin-transform-arrow-functions@npm:^7.0.0-0, @babel/plugin-transform-arrow-functions@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-arrow-functions@npm:7.29.7" dependencies: @@ -1357,6 +1485,19 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-async-generator-functions@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-async-generator-functions@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-remap-async-to-generator": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/efeeb26e8474d75b469b68fd7de74b757929b78aba5714ccb705a42f23d4e0de178ca09b59efd19113d42461bcf876dd9a919fd61f4e5e6fd1d1e5e7998e5bfe + languageName: node + linkType: hard + "@babel/plugin-transform-async-to-generator@npm:^7.24.7, @babel/plugin-transform-async-to-generator@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-async-to-generator@npm:7.27.1" @@ -1383,6 +1524,19 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-async-to-generator@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-async-to-generator@npm:7.29.7" + dependencies: + "@babel/helper-module-imports": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-remap-async-to-generator": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/1aa514d28a2a87747f54e031cf6d2884a792a41c8bb9ebcf4d3d663284a4ae14e02c744ad76819ab09b96b6a0cdb60dd20e54c75f2eb9c3cc3fb255e0ef79e74 + languageName: node + linkType: hard + "@babel/plugin-transform-block-scoped-functions@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-block-scoped-functions@npm:7.27.1" @@ -1394,6 +1548,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-block-scoped-functions@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-block-scoped-functions@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/bb790629b9bce5215f932932222b50f371f8dfd30b874b7e6ea294213ddf10f427d5fc80dc7e1c0fba3568f02c1865290ce40aef89657570d98c4eb13b6af3c2 + languageName: node + linkType: hard + "@babel/plugin-transform-block-scoping@npm:^7.25.0, @babel/plugin-transform-block-scoping@npm:^7.28.5": version: 7.28.5 resolution: "@babel/plugin-transform-block-scoping@npm:7.28.5" @@ -1416,7 +1581,18 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-class-properties@npm:^7.0.0-0": +"@babel/plugin-transform-block-scoping@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-block-scoping@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/ec4e45aefd4e1d276d0ee143bc521c2a3b38e59cc7dcef014d43c9a844416160d89f98953399e69e547a5743474d0986cb62155514109eab73b942beeb453a3f + languageName: node + linkType: hard + +"@babel/plugin-transform-class-properties@npm:^7.0.0-0, @babel/plugin-transform-class-properties@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-class-properties@npm:7.29.7" dependencies: @@ -1476,7 +1652,19 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-classes@npm:^7.0.0-0": +"@babel/plugin-transform-class-static-block@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-class-static-block@npm:7.29.7" + dependencies: + "@babel/helper-create-class-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.12.0 + checksum: 10c0/d2fa7e8af5d05cee838bab20b624c2911ef6618c7ead146f6ace6421280d0e57487fc2b946b42832289a35764b559e584983b32e51bf5a85596495ba3452970f + languageName: node + linkType: hard + +"@babel/plugin-transform-classes@npm:^7.0.0-0, @babel/plugin-transform-classes@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-classes@npm:7.29.7" dependencies: @@ -1548,6 +1736,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-computed-properties@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-computed-properties@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/template": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a8755338ffbfb374bafc2b3c22b00b025b8e5cf1cbac9c1ecdb3fb4c538508cb1b8f7a4e8c874b05df5166ff19ef105e65d54fefcf0546c628fa67214453b708 + languageName: node + linkType: hard + "@babel/plugin-transform-destructuring@npm:^7.24.8, @babel/plugin-transform-destructuring@npm:^7.28.0, @babel/plugin-transform-destructuring@npm:^7.28.5": version: 7.28.5 resolution: "@babel/plugin-transform-destructuring@npm:7.28.5" @@ -1560,6 +1760,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-destructuring@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-destructuring@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/74ff73303d32f8379f636741d3e48e2a20bbeb6e8b0d9343daad1952242f0ea78f319b4c871b5623739033b61459c9eed3d98f699fd192c45eab61ed8ad4c5ec + languageName: node + linkType: hard + "@babel/plugin-transform-dotall-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-dotall-regex@npm:7.27.1" @@ -1584,6 +1796,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-dotall-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-dotall-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/1c3eecc214bae152fc9af66b6d7e045a58e364a1cbc18a6c8e0ecea2d470eb6171f35b454dde51dffb4e687245bc8ad9abb9e233cc708db5bd3bdced74a1511c + languageName: node + linkType: hard + "@babel/plugin-transform-duplicate-keys@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-duplicate-keys@npm:7.27.1" @@ -1595,6 +1819,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-duplicate-keys@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-duplicate-keys@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/0d895326d8b29f6acaa8da72ebdc6393537311eaa33d8cab99cbb322a73c21be51d5d932a6d0c124e4f51539c5731dc3ed93084d3a2078a63db59dda6b00a724 + languageName: node + linkType: hard + "@babel/plugin-transform-duplicate-named-capturing-groups-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-duplicate-named-capturing-groups-regex@npm:7.27.1" @@ -1619,6 +1854,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-duplicate-named-capturing-groups-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-duplicate-named-capturing-groups-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/1dfecfb693ad6f1b6b779ac2fd676df048a051b83b4856f9ddced6217cd1f69b96259b01b5a67ee970fe531edf845944aadcc3f5dc74780331997f1dbf2de0da + languageName: node + linkType: hard + "@babel/plugin-transform-dynamic-import@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-dynamic-import@npm:7.27.1" @@ -1630,6 +1877,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-dynamic-import@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-dynamic-import@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/9f824556ab369173e5945cceeb3b83516a2896b161a5cd9f14641e30b7d1535426eebbf04d1f3cfcac557e0148180650584b4ce552b09a6b03f03adf1fbc689c + languageName: node + linkType: hard + "@babel/plugin-transform-explicit-resource-management@npm:^7.28.0": version: 7.28.0 resolution: "@babel/plugin-transform-explicit-resource-management@npm:7.28.0" @@ -1654,6 +1912,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-explicit-resource-management@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-explicit-resource-management@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/plugin-transform-destructuring": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/083ef2bbc8c78ff80d70b1fe12604a8a2cc6a5ec68115c6efa4be7b5291b7576a092a102739af7c7e743cad79234b7cb7efe4216ca1913b598e0afc04a9962d5 + languageName: node + linkType: hard + "@babel/plugin-transform-exponentiation-operator@npm:^7.28.5": version: 7.28.5 resolution: "@babel/plugin-transform-exponentiation-operator@npm:7.28.5" @@ -1676,6 +1946,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-exponentiation-operator@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-exponentiation-operator@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/1db57e065c5bceafcf7839d0c4cefd5723adba6d858df89d077d3f336364266a2dab71f1eb44746c14024736dd15fbe20ea392128c16b898abefc27cc1ca173f + languageName: node + linkType: hard + "@babel/plugin-transform-export-namespace-from@npm:^7.23.4, @babel/plugin-transform-export-namespace-from@npm:^7.25.9, @babel/plugin-transform-export-namespace-from@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-export-namespace-from@npm:7.27.1" @@ -1687,6 +1968,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-export-namespace-from@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-export-namespace-from@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/6cd951e366c5c30223c409157e312d949feecfae8b4b4927053764ec71ff2bacf43aceda9b53239cd90d71caf4ae849c70549e0d3e31377dd8f42a0c283bdda2 + languageName: node + linkType: hard + "@babel/plugin-transform-flow-strip-types@npm:^7.25.2, @babel/plugin-transform-flow-strip-types@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-flow-strip-types@npm:7.27.1" @@ -1699,6 +1991,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-flow-strip-types@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-flow-strip-types@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/plugin-syntax-flow": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/083e864a60927cde7bd75822bdf5c6c72de200ee955e48f6ff5923f0d2b0744ab8f1b64c45e74b88ae3796f03721489afffdb0536f25c54d0dd58508a8904f40 + languageName: node + linkType: hard + "@babel/plugin-transform-for-of@npm:^7.24.7, @babel/plugin-transform-for-of@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-for-of@npm:7.27.1" @@ -1711,6 +2015,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-for-of@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-for-of@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-skip-transparent-expression-wrappers": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/1d0143067df5f0d5ff0097099876da5d9d0fb7e2670939fde2523a0b14be2ea2cbcf736f874081d13ec5c3fca756f7aa1782a7c8cf17c262b0a493115a23b6b1 + languageName: node + linkType: hard + "@babel/plugin-transform-function-name@npm:^7.25.1, @babel/plugin-transform-function-name@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-function-name@npm:7.27.1" @@ -1724,6 +2040,19 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-function-name@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-function-name@npm:7.29.7" + dependencies: + "@babel/helper-compilation-targets": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/3ed2bca4f49356cd8fe1aa69daf5de63451be3b9271264eae536baf8d53476c63033e7fed6b5e97277cc10bdbfa10148f2f03315ca4cd94fae2b4ad5cb9b437f + languageName: node + linkType: hard + "@babel/plugin-transform-json-strings@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-json-strings@npm:7.27.1" @@ -1746,6 +2075,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-json-strings@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-json-strings@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/6cc66ffbfa1dd50f1f225da0668740e93357ea84e67c798e9814085595d4f04a65d0d1368d15fd4b0022b7e76eded3a1c822791a93a8855b62897c836c547fff + languageName: node + linkType: hard + "@babel/plugin-transform-literals@npm:^7.25.2, @babel/plugin-transform-literals@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-literals@npm:7.27.1" @@ -1757,6 +2097,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-literals@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-literals@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/bcfb8ca4092f2927ed61fdb75ddbadd701eda08ea2dd972f110d2ef1428643275c7adc7ce26847e15fbd573997e93654ff9afb4c1767a058e6d5843cbb9a4ae7 + languageName: node + linkType: hard + "@babel/plugin-transform-logical-assignment-operators@npm:^7.24.7, @babel/plugin-transform-logical-assignment-operators@npm:^7.28.5": version: 7.28.5 resolution: "@babel/plugin-transform-logical-assignment-operators@npm:7.28.5" @@ -1779,6 +2130,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-logical-assignment-operators@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-logical-assignment-operators@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/b0b85f47bde7efd253b273bcbe317178df6f281b99f8399c04a3af1a8b0878ddb6e3d57e395292917d0376c337e57f4d64ad9f861001590a90f888c30abf2c51 + languageName: node + linkType: hard + "@babel/plugin-transform-member-expression-literals@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-member-expression-literals@npm:7.27.1" @@ -1790,6 +2152,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-member-expression-literals@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-member-expression-literals@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/b8db313de0a4aeb3a617afcf9413774a53ec71a361030c89dbe2d05aa17310e9d33d4307727c898bfc9b07a9c676482fa8b0463840d1829c21323bc04623d556 + languageName: node + linkType: hard + "@babel/plugin-transform-modules-amd@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-modules-amd@npm:7.27.1" @@ -1802,6 +2175,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-modules-amd@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-modules-amd@npm:7.29.7" + dependencies: + "@babel/helper-module-transforms": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/425e9f99506968196a239944fe4eb51e144eadc2490b49a74798f92226f76b328d13b13e90e07962cd5df327994011570a3c2883b65091dde984b5bdff066c6b + languageName: node + linkType: hard + "@babel/plugin-transform-modules-commonjs@npm:^7.24.8, @babel/plugin-transform-modules-commonjs@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-modules-commonjs@npm:7.27.1" @@ -1866,6 +2251,20 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-modules-systemjs@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-modules-systemjs@npm:7.29.7" + dependencies: + "@babel/helper-module-transforms": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-validator-identifier": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/7b950bcc4a4b077b742b9299c8c9d4285e15854e23816d0b1c1177fafda4c153f3c3c4487c1b965afbf7a69a8788fc75656d0c591b91f0a5a543faabe738ca58 + languageName: node + linkType: hard + "@babel/plugin-transform-modules-umd@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-modules-umd@npm:7.27.1" @@ -1878,6 +2277,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-modules-umd@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-modules-umd@npm:7.29.7" + dependencies: + "@babel/helper-module-transforms": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/14545c7dfc8f95010766075d9cd4c1bf99a868c2dad7f780e9a609c6d94d23044f85672548b35a2162c027647386c0cfb85f68cee8f4e02352683acf6aade76d + languageName: node + linkType: hard + "@babel/plugin-transform-named-capturing-groups-regex@npm:^7.24.7, @babel/plugin-transform-named-capturing-groups-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-named-capturing-groups-regex@npm:7.27.1" @@ -1902,6 +2313,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-named-capturing-groups-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-named-capturing-groups-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/e16e270fc3640baf403497cbbab196cc0f18477576cf3535713c851f69c6e27b2902cc7502c3a863dce7f0432dc344ddcb14766a2af7cf37333aa864c7e5739b + languageName: node + linkType: hard + "@babel/plugin-transform-new-target@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-new-target@npm:7.27.1" @@ -1913,7 +2336,18 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.0.0-0": +"@babel/plugin-transform-new-target@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-new-target@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/9a192ded7083850d5b3c5629a3da4fe209298ac9d7a84d1495916a5c841cd4017e4d126d98a3b7c322eafae3851345fe2670377a16561b9cbd817e952f6fdbd5 + languageName: node + linkType: hard + +"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.0.0-0, @babel/plugin-transform-nullish-coalescing-operator@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.29.7" dependencies: @@ -1968,6 +2402,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-numeric-separator@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-numeric-separator@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a0e79a9627717277cc21ede56d8e57da0ac5e0c9620c963da2526791657044b57eeeafe27f297754cfdea470dd590c763e9bc71d589f1850654f77f5f4f77ece + languageName: node + linkType: hard + "@babel/plugin-transform-object-rest-spread@npm:^7.24.7, @babel/plugin-transform-object-rest-spread@npm:^7.28.4": version: 7.28.4 resolution: "@babel/plugin-transform-object-rest-spread@npm:7.28.4" @@ -1998,6 +2443,21 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-object-rest-spread@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-object-rest-spread@npm:7.29.7" + dependencies: + "@babel/helper-compilation-targets": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/plugin-transform-destructuring": "npm:^7.29.7" + "@babel/plugin-transform-parameters": "npm:^7.29.7" + "@babel/traverse": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/7963bcbb3165699cabad1ac5dcf03c26ffbe41c4a130283376d6e7a69ee6a353105c666e533e024bdf28862968434d1e13635846c8d8e7f5f9271407112aed1c + languageName: node + linkType: hard + "@babel/plugin-transform-object-super@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-object-super@npm:7.27.1" @@ -2010,6 +2470,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-object-super@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-object-super@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-replace-supers": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/eacfa0f571cb9bbdb283253b41c7a3cafe03e6da81ea92f61d003971b3ada43a3f65e5392d31ba3fe7da6cd8b4210968729769f22d3f0feb418db9e66f167413 + languageName: node + linkType: hard + "@babel/plugin-transform-optional-catch-binding@npm:^7.24.7, @babel/plugin-transform-optional-catch-binding@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-optional-catch-binding@npm:7.27.1" @@ -2032,7 +2504,18 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-optional-chaining@npm:^7.0.0-0": +"@babel/plugin-transform-optional-catch-binding@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-optional-catch-binding@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/0df7a9cdaec349e4b893cb26b7ad45454b3ac01de722ccea5e8b4332d15f3e1bc2c130a18367052ce195c957178ad773121c5d586454c438d890585fc548dacc + languageName: node + linkType: hard + +"@babel/plugin-transform-optional-chaining@npm:^7.0.0-0, @babel/plugin-transform-optional-chaining@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-optional-chaining@npm:7.29.7" dependencies: @@ -2079,6 +2562,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-parameters@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-parameters@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/ea1ff347e7b33b2483d18dcb9fb6c58f9819312f38235bf142e12f5355fcf77bb6640929734b12bd26b900c7abbb8cbd77ad06082d4c10d6e0e097ad016237e6 + languageName: node + linkType: hard + "@babel/plugin-transform-private-methods@npm:^7.24.7, @babel/plugin-transform-private-methods@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-private-methods@npm:7.27.1" @@ -2103,6 +2597,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-private-methods@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-private-methods@npm:7.29.7" + dependencies: + "@babel/helper-create-class-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/d0dc12fa478d05e346dfb02fad2ed99b308d9a7324bada71530a62dc1ccbf07b4c2581ac677b7196b3994f191ef1922199e2c8f33957b726e2019ce07b4ced0f + languageName: node + linkType: hard + "@babel/plugin-transform-private-property-in-object@npm:^7.24.7, @babel/plugin-transform-private-property-in-object@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-private-property-in-object@npm:7.27.1" @@ -2129,6 +2635,19 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-private-property-in-object@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-private-property-in-object@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": "npm:^7.29.7" + "@babel/helper-create-class-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/6378b34725c6aab4b580cbb5d35ca45ba52213084d54c6672bc4282d86dc45937b8788ad450a9fb60ceb405e3c0d4b25e2804293c2509b54c5e0078babd56a8a + languageName: node + linkType: hard + "@babel/plugin-transform-property-literals@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-property-literals@npm:7.27.1" @@ -2140,6 +2659,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-property-literals@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-property-literals@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/231ef97caeed1b79676978c9c04f203ba39f98da79097b733946aa29eb302d7cefc863d407f032a805080e8d20f70d7b2265c9c3ce634ca627d63c8f0f6e6e42 + languageName: node + linkType: hard + "@babel/plugin-transform-react-display-name@npm:^7.24.7, @babel/plugin-transform-react-display-name@npm:^7.28.0": version: 7.28.0 resolution: "@babel/plugin-transform-react-display-name@npm:7.28.0" @@ -2151,6 +2681,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-react-display-name@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-react-display-name@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/0a3b2461b189d6f4ca4f691bb4d1872bdebbac90d2e933a42a2fb22a0d26c81fa1ba7bc8128f3886b3f0f8a0ffe5aa0d7eaf4cd5b9610fc4967913a060501781 + languageName: node + linkType: hard + "@babel/plugin-transform-react-jsx-development@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-react-jsx-development@npm:7.27.1" @@ -2162,6 +2703,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-react-jsx-development@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-react-jsx-development@npm:7.29.7" + dependencies: + "@babel/plugin-transform-react-jsx": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a2d44d068d35f8e6bc0437ba0da86526d4473491a43108caf77cba467a3ab522cbd09bcd8184b7a49f3d2b52e0883ad797ed2f91c090b857a80a6f383e88f449 + languageName: node + linkType: hard + "@babel/plugin-transform-react-jsx-self@npm:^7.24.7": version: 7.27.1 resolution: "@babel/plugin-transform-react-jsx-self@npm:7.27.1" @@ -2221,6 +2773,21 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-react-jsx@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-react-jsx@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": "npm:^7.29.7" + "@babel/helper-module-imports": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/plugin-syntax-jsx": "npm:^7.29.7" + "@babel/types": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/ae6487c3deafcffda8a2c1b1eb77e0b7121630fa1a9646cecd73dbbdd8271532a0f7f0e8c01f69b6d39a36d8d79eb67e2d51031369c9055f18f7d8e70d9b5446 + languageName: node + linkType: hard + "@babel/plugin-transform-react-pure-annotations@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-react-pure-annotations@npm:7.27.1" @@ -2233,6 +2800,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-react-pure-annotations@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-react-pure-annotations@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/ad85d57dfa105cd19dc5271f68c9a3e134ab96edce9b8c6b521fdb158499bc616df9d4ee9b386e9dc5532e67bd5682ba2047b88550699d7f1060eaaba80fb6d1 + languageName: node + linkType: hard + "@babel/plugin-transform-regenerator@npm:^7.24.7, @babel/plugin-transform-regenerator@npm:^7.28.4": version: 7.28.4 resolution: "@babel/plugin-transform-regenerator@npm:7.28.4" @@ -2248,10 +2827,21 @@ __metadata: version: 7.29.0 resolution: "@babel/plugin-transform-regenerator@npm:7.29.0" dependencies: - "@babel/helper-plugin-utils": "npm:^7.28.6" + "@babel/helper-plugin-utils": "npm:^7.28.6" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/86c7db9b97f85ee47c0fae0528802cbc06e5775e61580ee905335c16bb971270086764a3859873d9adcd7d0f913a5b93eb0dc271aec8fb9e93e090e4ac95e29e + languageName: node + linkType: hard + +"@babel/plugin-transform-regenerator@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-regenerator@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 10c0/86c7db9b97f85ee47c0fae0528802cbc06e5775e61580ee905335c16bb971270086764a3859873d9adcd7d0f913a5b93eb0dc271aec8fb9e93e090e4ac95e29e + checksum: 10c0/b991ab501c1c2c323398054211f8cd992f1db438b5b5d548d63dc74ff9591d52a50385160fdba2b62aae4f9610a321355a8ff911f1c4bd80dc74b555cad61468 languageName: node linkType: hard @@ -2279,6 +2869,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-regexp-modifiers@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-regexp-modifiers@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/1c6c79f87a7163d33c1c9d787d239e3981755a66822ef0d41a95ce12e76277a247eaeeab29e3cf79bb6288e37e48a18accc13c3a9c351c06e4303b1d9b8b37cb + languageName: node + linkType: hard + "@babel/plugin-transform-reserved-words@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-reserved-words@npm:7.27.1" @@ -2290,6 +2892,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-reserved-words@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-reserved-words@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/9eee586b7c7a9a34a659fd4d55ae8b156b03c8120a8459724062c212a59327ee8878168c0ce6bb5abd6c2a28aa87bc280b5180b1cbb2b03b12f7c71690f48718 + languageName: node + linkType: hard + "@babel/plugin-transform-runtime@npm:^7.24.7": version: 7.28.5 resolution: "@babel/plugin-transform-runtime@npm:7.28.5" @@ -2306,7 +2919,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-shorthand-properties@npm:^7.0.0-0": +"@babel/plugin-transform-shorthand-properties@npm:^7.0.0-0, @babel/plugin-transform-shorthand-properties@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-shorthand-properties@npm:7.29.7" dependencies: @@ -2352,6 +2965,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-spread@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-spread@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-skip-transparent-expression-wrappers": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a3a5639eac8cc4a9cb3d8b2098a0a341b36e3ae11d29729cac1042d07a31b5290c32fa2c12cd2f122bf581bd0a65fec6b7b6c7446eef5a81e657739a579c0d5c + languageName: node + linkType: hard + "@babel/plugin-transform-sticky-regex@npm:^7.24.7, @babel/plugin-transform-sticky-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-sticky-regex@npm:7.27.1" @@ -2363,7 +2988,18 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-template-literals@npm:^7.0.0-0": +"@babel/plugin-transform-sticky-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-sticky-regex@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/45b245f07841cc30a8e781a77bb09a2e86b2cc7f872785f315c92e2805825be43ac99f3ba63afcd4722307c648cb7d3f4f6f3e2b27d2d3e281414532a2601762 + languageName: node + linkType: hard + +"@babel/plugin-transform-template-literals@npm:^7.0.0-0, @babel/plugin-transform-template-literals@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-template-literals@npm:7.29.7" dependencies: @@ -2396,6 +3032,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-typeof-symbol@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-typeof-symbol@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/937408e0b9b2c8df6a6ce590421096fd793f77b417eb1f3f5ec560362e0a855d6ef66346c12cc7b337b8fa1d3ecf6b9b15674aa5ded54141adaed4cdeeed8528 + languageName: node + linkType: hard + "@babel/plugin-transform-typescript@npm:^7.25.2, @babel/plugin-transform-typescript@npm:^7.28.5": version: 7.28.5 resolution: "@babel/plugin-transform-typescript@npm:7.28.5" @@ -2437,6 +3084,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-unicode-escapes@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-unicode-escapes@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/65090012ede685913fb1020a585361dac366801d87caa577075924cac3c3ddd8e2a953bc6f75048dd480e62e529482e6ba68b945b0780087981c9ffff32e84f2 + languageName: node + linkType: hard + "@babel/plugin-transform-unicode-property-regex@npm:^7.27.1": version: 7.27.1 resolution: "@babel/plugin-transform-unicode-property-regex@npm:7.27.1" @@ -2461,7 +3119,19 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-unicode-regex@npm:^7.0.0-0": +"@babel/plugin-transform-unicode-property-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-unicode-property-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/3a869cab02013209192da67ce911fa577eed03293331ee1d2c98498461f31fb5e99cbcb47f0c1c3211cf0c48fdd391060c77ac6a8f9978cd1f48e1096024cb73 + languageName: node + linkType: hard + +"@babel/plugin-transform-unicode-regex@npm:^7.0.0-0, @babel/plugin-transform-unicode-regex@npm:^7.29.7": version: 7.29.7 resolution: "@babel/plugin-transform-unicode-regex@npm:7.29.7" dependencies: @@ -2509,6 +3179,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-unicode-sets-regex@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-unicode-sets-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 10c0/7e07f6435b2ba32de80460ddcacc4214c9380984c00fd41ddaa79e4df3f06c022c1283e86bcae7ee09081f4e020f0bb76b26b9f98dc5ea7f4647f559544fe5f9 + languageName: node + linkType: hard + "@babel/preset-env@npm:^7.12.11": version: 7.29.5 resolution: "@babel/preset-env@npm:7.29.5" @@ -2590,6 +3272,87 @@ __metadata: languageName: node linkType: hard +"@babel/preset-env@npm:^7.18.2": + version: 7.29.7 + resolution: "@babel/preset-env@npm:7.29.7" + dependencies: + "@babel/compat-data": "npm:^7.29.7" + "@babel/helper-compilation-targets": "npm:^7.29.7" + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-validator-option": "npm:^7.29.7" + "@babel/plugin-bugfix-firefox-class-in-computed-class-key": "npm:^7.29.7" + "@babel/plugin-bugfix-safari-class-field-initializer-scope": "npm:^7.29.7" + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "npm:^7.29.7" + "@babel/plugin-bugfix-safari-rest-destructuring-rhs-array": "npm:^7.29.7" + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "npm:^7.29.7" + "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "npm:^7.29.7" + "@babel/plugin-proposal-private-property-in-object": "npm:7.21.0-placeholder-for-preset-env.2" + "@babel/plugin-syntax-import-assertions": "npm:^7.29.7" + "@babel/plugin-syntax-import-attributes": "npm:^7.29.7" + "@babel/plugin-syntax-unicode-sets-regex": "npm:^7.18.6" + "@babel/plugin-transform-arrow-functions": "npm:^7.29.7" + "@babel/plugin-transform-async-generator-functions": "npm:^7.29.7" + "@babel/plugin-transform-async-to-generator": "npm:^7.29.7" + "@babel/plugin-transform-block-scoped-functions": "npm:^7.29.7" + "@babel/plugin-transform-block-scoping": "npm:^7.29.7" + "@babel/plugin-transform-class-properties": "npm:^7.29.7" + "@babel/plugin-transform-class-static-block": "npm:^7.29.7" + "@babel/plugin-transform-classes": "npm:^7.29.7" + "@babel/plugin-transform-computed-properties": "npm:^7.29.7" + "@babel/plugin-transform-destructuring": "npm:^7.29.7" + "@babel/plugin-transform-dotall-regex": "npm:^7.29.7" + "@babel/plugin-transform-duplicate-keys": "npm:^7.29.7" + "@babel/plugin-transform-duplicate-named-capturing-groups-regex": "npm:^7.29.7" + "@babel/plugin-transform-dynamic-import": "npm:^7.29.7" + "@babel/plugin-transform-explicit-resource-management": "npm:^7.29.7" + "@babel/plugin-transform-exponentiation-operator": "npm:^7.29.7" + "@babel/plugin-transform-export-namespace-from": "npm:^7.29.7" + "@babel/plugin-transform-for-of": "npm:^7.29.7" + "@babel/plugin-transform-function-name": "npm:^7.29.7" + "@babel/plugin-transform-json-strings": "npm:^7.29.7" + "@babel/plugin-transform-literals": "npm:^7.29.7" + "@babel/plugin-transform-logical-assignment-operators": "npm:^7.29.7" + "@babel/plugin-transform-member-expression-literals": "npm:^7.29.7" + "@babel/plugin-transform-modules-amd": "npm:^7.29.7" + "@babel/plugin-transform-modules-commonjs": "npm:^7.29.7" + "@babel/plugin-transform-modules-systemjs": "npm:^7.29.7" + "@babel/plugin-transform-modules-umd": "npm:^7.29.7" + "@babel/plugin-transform-named-capturing-groups-regex": "npm:^7.29.7" + "@babel/plugin-transform-new-target": "npm:^7.29.7" + "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.29.7" + "@babel/plugin-transform-numeric-separator": "npm:^7.29.7" + "@babel/plugin-transform-object-rest-spread": "npm:^7.29.7" + "@babel/plugin-transform-object-super": "npm:^7.29.7" + "@babel/plugin-transform-optional-catch-binding": "npm:^7.29.7" + "@babel/plugin-transform-optional-chaining": "npm:^7.29.7" + "@babel/plugin-transform-parameters": "npm:^7.29.7" + "@babel/plugin-transform-private-methods": "npm:^7.29.7" + "@babel/plugin-transform-private-property-in-object": "npm:^7.29.7" + "@babel/plugin-transform-property-literals": "npm:^7.29.7" + "@babel/plugin-transform-regenerator": "npm:^7.29.7" + "@babel/plugin-transform-regexp-modifiers": "npm:^7.29.7" + "@babel/plugin-transform-reserved-words": "npm:^7.29.7" + "@babel/plugin-transform-shorthand-properties": "npm:^7.29.7" + "@babel/plugin-transform-spread": "npm:^7.29.7" + "@babel/plugin-transform-sticky-regex": "npm:^7.29.7" + "@babel/plugin-transform-template-literals": "npm:^7.29.7" + "@babel/plugin-transform-typeof-symbol": "npm:^7.29.7" + "@babel/plugin-transform-unicode-escapes": "npm:^7.29.7" + "@babel/plugin-transform-unicode-property-regex": "npm:^7.29.7" + "@babel/plugin-transform-unicode-regex": "npm:^7.29.7" + "@babel/plugin-transform-unicode-sets-regex": "npm:^7.29.7" + "@babel/preset-modules": "npm:0.1.6-no-external-plugins" + babel-plugin-polyfill-corejs2: "npm:^0.4.15" + babel-plugin-polyfill-corejs3: "npm:^0.14.0" + babel-plugin-polyfill-regenerator: "npm:^0.6.6" + core-js-compat: "npm:^3.48.0" + semver: "npm:^6.3.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a6527c75e54c06c453e214496df83c2f6aa61da32d786e9a8921af05c4bc580c87ee64248122652df8d0f4b140d651b11282cd3dc3b61bb640b2a86b5812eabf + languageName: node + linkType: hard + "@babel/preset-env@npm:^7.23.8": version: 7.28.5 resolution: "@babel/preset-env@npm:7.28.5" @@ -2683,6 +3446,19 @@ __metadata: languageName: node linkType: hard +"@babel/preset-flow@npm:^7.17.12": + version: 7.29.7 + resolution: "@babel/preset-flow@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-validator-option": "npm:^7.29.7" + "@babel/plugin-transform-flow-strip-types": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/17bb0f3c320fadb631ab64951ec0f6e05a62b15418c7864c181645b3205cabd3a8541dd3f497bf06cb1d0e04a3806f6cffee7675c97a87d4aaa04961f334cdcb + languageName: node + linkType: hard + "@babel/preset-modules@npm:0.1.6-no-external-plugins": version: 0.1.6-no-external-plugins resolution: "@babel/preset-modules@npm:0.1.6-no-external-plugins" @@ -2712,6 +3488,22 @@ __metadata: languageName: node linkType: hard +"@babel/preset-react@npm:^7.17.12": + version: 7.29.7 + resolution: "@babel/preset-react@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": "npm:^7.29.7" + "@babel/helper-validator-option": "npm:^7.29.7" + "@babel/plugin-transform-react-display-name": "npm:^7.29.7" + "@babel/plugin-transform-react-jsx": "npm:^7.29.7" + "@babel/plugin-transform-react-jsx-development": "npm:^7.29.7" + "@babel/plugin-transform-react-pure-annotations": "npm:^7.29.7" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 10c0/a84e90805ce06dd5d7fcbfd8e32fe0c79966feba218e1d89097699ff5224d232e56191688ae264be2c6ef516523e3e2246949439e1e141d21def881a5752181f + languageName: node + linkType: hard + "@babel/preset-typescript@npm:^7.12.7, @babel/preset-typescript@npm:^7.23.0, @babel/preset-typescript@npm:^7.23.3": version: 7.28.5 resolution: "@babel/preset-typescript@npm:7.28.5" @@ -2727,7 +3519,7 @@ __metadata: languageName: node linkType: hard -"@babel/preset-typescript@npm:^7.16.7, @babel/preset-typescript@npm:^7.28.5": +"@babel/preset-typescript@npm:^7.16.7, @babel/preset-typescript@npm:^7.17.12, @babel/preset-typescript@npm:^7.27.0, @babel/preset-typescript@npm:^7.28.5": version: 7.29.7 resolution: "@babel/preset-typescript@npm:7.29.7" dependencies: @@ -4557,6 +5349,8 @@ __metadata: version: 0.0.0-use.local resolution: "@fishjam-cloud/react-native-vision-camera-source@workspace:packages/react-native-vision-camera-source" dependencies: + "@babel/core": "npm:^7.29.0" + "@babel/preset-typescript": "npm:^7.27.0" "@fishjam-cloud/react-native-client": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*" "@types/react": "npm:19.2.14" @@ -4566,12 +5360,15 @@ __metadata: eslint-plugin-prettier: "npm:^5.5.1" react: "npm:19.2.3" react-native: "npm:0.85.3" + react-native-builder-bob: "npm:^0.18.2" react-native-nitro-modules: "npm:0.35.7" react-native-vision-camera: "npm:5.0.10" react-native-vision-camera-worklets: "npm:5.0.10" react-native-webgpu: "npm:0.5.15" react-native-worklets: "npm:>=0.9.0" + typegpu: "npm:^0.11.8" typescript: "npm:^5.8.3" + unplugin-typegpu: "npm:^0.11.5" peerDependencies: "@fishjam-cloud/react-native-client": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*" @@ -5127,6 +5924,13 @@ __metadata: languageName: node linkType: hard +"@jridgewell/sourcemap-codec@npm:^1.5.5": + version: 1.5.5 + resolution: "@jridgewell/sourcemap-codec@npm:1.5.5" + checksum: 10c0/f9e538f302b63c0ebc06eecb1dd9918dd4289ed36147a0ddce35d6ea4d7ebbda243cda7b2213b6a5e1d8087a298d5cf630fb2bd39329cdecb82017023f6081a0 + languageName: node + linkType: hard + "@jridgewell/trace-mapping@npm:^0.3.18, @jridgewell/trace-mapping@npm:^0.3.28": version: 0.3.31 resolution: "@jridgewell/trace-mapping@npm:0.3.31" @@ -8457,6 +9261,16 @@ __metadata: languageName: node linkType: hard +"ast-kit@npm:^2.2.0": + version: 2.2.0 + resolution: "ast-kit@npm:2.2.0" + dependencies: + "@babel/parser": "npm:^7.28.5" + pathe: "npm:^2.0.3" + checksum: 10c0/d885f3a4e9837e730451a667d26936eef34773d6e5ecacd771a3e9d1f82fdc45d38958ab35e18880e0cf667896604a599497c5186f2578cf73c0d802ed7fc697 + languageName: node + linkType: hard + "async-function@npm:^1.0.0": version: 1.0.0 resolution: "async-function@npm:1.0.0" @@ -8944,6 +9758,15 @@ __metadata: languageName: node linkType: hard +"baseline-browser-mapping@npm:^2.10.42": + version: 2.10.42 + resolution: "baseline-browser-mapping@npm:2.10.42" + bin: + baseline-browser-mapping: dist/cli.cjs + checksum: 10c0/4a9f54818d17d8dbee69096563b2cd5e2175f66752c479f1c10d31e072c1c2b7241a9bdaee78d5236178c581ca2735e75fbb0d0877d6492958eed9f6e46e03f3 + languageName: node + linkType: hard + "baseline-browser-mapping@npm:^2.8.25": version: 2.8.32 resolution: "baseline-browser-mapping@npm:2.8.32" @@ -9084,6 +9907,21 @@ __metadata: languageName: node linkType: hard +"browserslist@npm:^4.20.4": + version: 4.28.5 + resolution: "browserslist@npm:4.28.5" + dependencies: + baseline-browser-mapping: "npm:^2.10.42" + caniuse-lite: "npm:^1.0.30001800" + electron-to-chromium: "npm:^1.5.387" + node-releases: "npm:^2.0.50" + update-browserslist-db: "npm:^1.2.3" + bin: + browserslist: cli.js + checksum: 10c0/d6bb4ab286a7071db52cbeabd46ff816e09c9171d7e5da246f0b9489601ad3d4adb9fc3d14fa934a8646078f8a717507c0518a364128735a3b5a7875900b5a19 + languageName: node + linkType: hard + "browserslist@npm:^4.24.0, browserslist@npm:^4.24.4": version: 4.24.4 resolution: "browserslist@npm:4.24.4" @@ -9320,6 +10158,13 @@ __metadata: languageName: node linkType: hard +"caniuse-lite@npm:^1.0.30001800": + version: 1.0.30001803 + resolution: "caniuse-lite@npm:1.0.30001803" + checksum: 10c0/71586e9c84633cf766b208448eb76f860ec6e3befffc626d1f004e1902da063a7ab96cc94d1f4b36c0324e12997b3325a726de3287edfec91d0df495627a1d43 + languageName: node + linkType: hard + "case-anything@npm:^2.1.13": version: 2.1.13 resolution: "case-anything@npm:2.1.13" @@ -9840,7 +10685,7 @@ __metadata: languageName: node linkType: hard -"cosmiconfig@npm:^7.0.0": +"cosmiconfig@npm:^7.0.0, cosmiconfig@npm:^7.0.1": version: 7.1.0 resolution: "cosmiconfig@npm:7.1.0" dependencies: @@ -10294,7 +11139,14 @@ __metadata: languageName: node linkType: hard -"del@npm:^6.0.0": +"defu@npm:^6.1.4": + version: 6.1.7 + resolution: "defu@npm:6.1.7" + checksum: 10c0/e6635388103c8be3c574ac31302f6930e5e6eeedba32cb1b30cf993c7d9fb571aec2485446dfa23bfa63e55e66156fe109027a9695db82a50f931e91e8d4bedb + languageName: node + linkType: hard + +"del@npm:^6.0.0, del@npm:^6.1.1": version: 6.1.1 resolution: "del@npm:6.1.1" dependencies: @@ -10525,6 +11377,13 @@ __metadata: languageName: node linkType: hard +"electron-to-chromium@npm:^1.5.387": + version: 1.5.389 + resolution: "electron-to-chromium@npm:1.5.389" + checksum: 10c0/3681a3245fef5dfc8d9cdee231e80f8fe4fd7a87713d3a29a21ae2ec0cb6995cf14f3b3fd9e3fa8b483256e28ba2fa1daa9f23e93911a3a7241cd8f5daee9665 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.141 resolution: "electron-to-chromium@npm:1.5.141" @@ -12486,6 +13345,17 @@ __metadata: languageName: node linkType: hard +"fs-extra@npm:^10.1.0": + version: 10.1.0 + resolution: "fs-extra@npm:10.1.0" + dependencies: + graceful-fs: "npm:^4.2.0" + jsonfile: "npm:^6.0.1" + universalify: "npm:^2.0.0" + checksum: 10c0/5f579466e7109719d162a9249abbeffe7f426eb133ea486e020b89bc6d67a741134076bf439983f2eb79276ceaf6bd7b7c1e43c3fd67fe889863e69072fb0a5e + languageName: node + linkType: hard + "fs-extra@npm:^9.0.1": version: 9.1.0 resolution: "fs-extra@npm:9.1.0" @@ -12782,6 +13652,19 @@ __metadata: languageName: node linkType: hard +"glob@npm:^8.0.3": + version: 8.1.0 + resolution: "glob@npm:8.1.0" + dependencies: + fs.realpath: "npm:^1.0.0" + inflight: "npm:^1.0.4" + inherits: "npm:2" + minimatch: "npm:^5.0.1" + once: "npm:^1.3.0" + checksum: 10c0/cb0b5cab17a59c57299376abe5646c7070f8acb89df5595b492dba3bfb43d301a46c01e5695f01154e6553168207cb60d4eaf07d3be4bc3eb9b0457c5c561d0f + languageName: node + linkType: hard + "glob@npm:^9.3.3": version: 9.3.5 resolution: "glob@npm:9.3.5" @@ -14132,6 +15015,17 @@ __metadata: languageName: node linkType: hard +"jetifier@npm:^2.0.0": + version: 2.0.0 + resolution: "jetifier@npm:2.0.0" + bin: + jetifier: bin/jetify + jetifier-standalone: bin/jetifier-standalone + jetify: bin/jetify + checksum: 10c0/3716b7a26d1d912aef604a88fdd4aface01b26db95873860adecdb5ed4d1396f5a847efe5315630a90b2e84f447c872dedf55f368482fc9e59236edca03ff100 + languageName: node + linkType: hard + "jimp-compact@npm:0.16.1": version: 0.16.1 resolution: "jimp-compact@npm:0.16.1" @@ -14310,7 +15204,7 @@ __metadata: languageName: node linkType: hard -"json5@npm:^2.1.3, json5@npm:^2.2.3": +"json5@npm:^2.1.3, json5@npm:^2.2.1, json5@npm:^2.2.3": version: 2.2.3 resolution: "json5@npm:2.2.3" bin: @@ -14360,6 +15254,13 @@ __metadata: languageName: node linkType: hard +"kleur@npm:^4.1.4": + version: 4.1.5 + resolution: "kleur@npm:4.1.5" + checksum: 10c0/e9de6cb49657b6fa70ba2d1448fd3d691a5c4370d8f7bbf1c2f64c24d461270f2117e1b0afe8cb3114f13bbd8e51de158c2a224953960331904e636a5e4c0f2a + languageName: node + linkType: hard + "lan-network@npm:^0.2.1": version: 0.2.1 resolution: "lan-network@npm:0.2.1" @@ -14799,6 +15700,15 @@ __metadata: languageName: node linkType: hard +"magic-string@npm:^0.30.21": + version: 0.30.21 + resolution: "magic-string@npm:0.30.21" + dependencies: + "@jridgewell/sourcemap-codec": "npm:^1.5.5" + checksum: 10c0/299378e38f9a270069fc62358522ddfb44e94244baa0d6a8980ab2a9b2490a1d03b236b447eee309e17eb3bddfa482c61259d47960eb018a904f0ded52780c4a + languageName: node + linkType: hard + "magicast@npm:^0.3.5": version: 0.3.5 resolution: "magicast@npm:0.3.5" @@ -15491,6 +16401,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.9 + resolution: "minimatch@npm:5.1.9" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/4202718683815a7288b13e470160a4f9560cf392adef4f453927505817e01ef6b3476ecde13cfcaed17e7326dd3b69ad44eb2daeb19a217c5500f9277893f1d6 + languageName: node + linkType: hard + "minimatch@npm:^5.1.0": version: 5.1.6 resolution: "minimatch@npm:5.1.6" @@ -15837,6 +16756,13 @@ __metadata: languageName: node linkType: hard +"node-releases@npm:^2.0.50": + version: 2.0.51 + resolution: "node-releases@npm:2.0.51" + checksum: 10c0/6cf3fe1f9eabe02ce6de67e9db77b73edc9f5625003791e1f8f239f8e2a851450a5aeb1eff2cc43cfb26312e19b26bfe07626bf428479e6a76f56553fc6148da + languageName: node + linkType: hard + "nopt@npm:^8.0.0": version: 8.1.0 resolution: "nopt@npm:8.1.0" @@ -16729,7 +17655,7 @@ __metadata: languageName: node linkType: hard -"prompts@npm:^2.2.1, prompts@npm:^2.3.2, prompts@npm:^2.4.0": +"prompts@npm:^2.2.1, prompts@npm:^2.3.2, prompts@npm:^2.4.0, prompts@npm:^2.4.2": version: 2.4.2 resolution: "prompts@npm:2.4.2" dependencies: @@ -16993,6 +17919,39 @@ __metadata: languageName: node linkType: hard +"react-native-builder-bob@npm:^0.18.2": + version: 0.18.3 + resolution: "react-native-builder-bob@npm:0.18.3" + dependencies: + "@babel/core": "npm:^7.18.5" + "@babel/plugin-proposal-class-properties": "npm:^7.17.12" + "@babel/preset-env": "npm:^7.18.2" + "@babel/preset-flow": "npm:^7.17.12" + "@babel/preset-react": "npm:^7.17.12" + "@babel/preset-typescript": "npm:^7.17.12" + browserslist: "npm:^4.20.4" + cosmiconfig: "npm:^7.0.1" + cross-spawn: "npm:^7.0.3" + dedent: "npm:^0.7.0" + del: "npm:^6.1.1" + fs-extra: "npm:^10.1.0" + glob: "npm:^8.0.3" + is-git-dirty: "npm:^2.0.1" + jetifier: "npm:^2.0.0" + json5: "npm:^2.2.1" + kleur: "npm:^4.1.4" + prompts: "npm:^2.4.2" + which: "npm:^2.0.2" + yargs: "npm:^17.5.1" + dependenciesMeta: + jetifier: + optional: true + bin: + bob: bin/bob + checksum: 10c0/a983284f71d1c06000e422287bffd3ba4f1d547c26681253cef42b1074cfd37b2b9ff8305cf3a1c7c3ee605fa959052ae430332151a5a97c7675fe6ef8f4cb63 + languageName: node + linkType: hard + "react-native-gesture-handler@npm:~2.28.0": version: 2.28.0 resolution: "react-native-gesture-handler@npm:2.28.0" @@ -19175,6 +20134,22 @@ __metadata: languageName: node linkType: hard +"tinyest-for-wgsl@npm:~0.3.2": + version: 0.3.3 + resolution: "tinyest-for-wgsl@npm:0.3.3" + dependencies: + tinyest: "npm:~0.3.1" + checksum: 10c0/83b1923dad518d1048f077d6418273cc40e9c905acc523de4d74df4ac2a290c29d6f7bf3c9518a810e2bc05f690718a52f1c21658404fc409405440cc3a9c0c1 + languageName: node + linkType: hard + +"tinyest@npm:~0.3.1, tinyest@npm:~0.3.2": + version: 0.3.2 + resolution: "tinyest@npm:0.3.2" + checksum: 10c0/05092ca73224eda2374ad0475522bf9946ab07854d2f6f09e48ebaf73f77a14e32e2be42f73191cef71320036438f8c8986b9a6844b99991d194d11c99f7df8b + languageName: node + linkType: hard + "tinyexec@npm:^0.3.2": version: 0.3.2 resolution: "tinyexec@npm:0.3.2" @@ -19465,6 +20440,13 @@ __metadata: languageName: node linkType: hard +"tsover-runtime@npm:^0.0.7": + version: 0.0.7 + resolution: "tsover-runtime@npm:0.0.7" + checksum: 10c0/62970f91cf18e7a3d2f6ca8ee688455a0326c97bf3ff1d9dccea547b67747e6b9210993dbfc680983e70150cab428ece112533b2430e46c53caf69561531d250 + languageName: node + linkType: hard + "tsup@npm:^8.4.0": version: 8.4.0 resolution: "tsup@npm:8.4.0" @@ -19603,6 +20585,13 @@ __metadata: languageName: node linkType: hard +"typed-binary@npm:^4.3.3": + version: 4.3.3 + resolution: "typed-binary@npm:4.3.3" + checksum: 10c0/fcb7806b430dc8cfc27476d069bdd6687d81066441efca4585d958351ec289fd09060f5182a4c788169b1586eceb78659ffa817727c4bcae8c3fe396a6b0ba6f + languageName: node + linkType: hard + "typed-emitter@npm:^2.1.0": version: 2.1.0 resolution: "typed-emitter@npm:2.1.0" @@ -19662,6 +20651,19 @@ __metadata: languageName: node linkType: hard +"typegpu@npm:^0.11.8": + version: 0.11.9 + resolution: "typegpu@npm:0.11.9" + dependencies: + tinyest: "npm:~0.3.2" + tsover-runtime: "npm:^0.0.7" + typed-binary: "npm:^4.3.3" + bin: + typegpu: ./bin.mjs + checksum: 10c0/44ba12e1c9af79849d4714390bef2b5521710749940bd2da4dd32d696869e21fc0c43f80beddad315c5605bfbb8910cd3d1a3a42efb26b2d8572265d8d1d5db7 + languageName: node + linkType: hard + "typescript@npm:^5.8.3": version: 5.8.3 resolution: "typescript@npm:5.8.3" @@ -19844,6 +20846,67 @@ __metadata: languageName: node linkType: hard +"unplugin-typegpu@npm:^0.11.5": + version: 0.11.6 + resolution: "unplugin-typegpu@npm:0.11.6" + dependencies: + "@babel/parser": "npm:^7.29.0" + "@babel/traverse": "npm:^7.29.0" + "@babel/types": "npm:^7.29.0" + ast-kit: "npm:^2.2.0" + defu: "npm:^6.1.4" + magic-string: "npm:^0.30.21" + pathe: "npm:^2.0.3" + picomatch: "npm:^4.0.4" + tinyest: "npm:~0.3.1" + tinyest-for-wgsl: "npm:~0.3.2" + unplugin: "npm:^3.0.0" + peerDependencies: + typegpu: ^0.11.9 + checksum: 10c0/1e93598f61c60ecec5c85c99617c65be01b7e1aa8dc2713a1ef5740bfb8f796e60ec8beb6edfad7f66a815c125b17c72fa41912ace543c6ece7e33f62fa15677 + languageName: node + linkType: hard + +"unplugin@npm:^3.0.0": + version: 3.3.0 + resolution: "unplugin@npm:3.3.0" + dependencies: + "@jridgewell/remapping": "npm:^2.3.5" + picomatch: "npm:^4.0.4" + webpack-virtual-modules: "npm:^0.6.2" + peerDependencies: + "@farmfe/core": "*" + "@rspack/core": "*" + bun-types-no-globals: "*" + esbuild: "*" + rolldown: "*" + rollup: "*" + unloader: "*" + vite: "*" + webpack: "*" + peerDependenciesMeta: + "@farmfe/core": + optional: true + "@rspack/core": + optional: true + bun-types-no-globals: + optional: true + esbuild: + optional: true + rolldown: + optional: true + rollup: + optional: true + unloader: + optional: true + vite: + optional: true + webpack: + optional: true + checksum: 10c0/86ee7c7fde40ffa8260ddeba5da51800e35c6cdd41ee8a556ec3827357b5d2ee13dcbe4ea6820451bc6c2b05b35165f96c7bf20662114af5ed52183a74886404 + languageName: node + linkType: hard + "unrs-resolver@npm:^1.6.2": version: 1.11.1 resolution: "unrs-resolver@npm:1.11.1" @@ -20427,6 +21490,13 @@ __metadata: languageName: node linkType: hard +"webpack-virtual-modules@npm:^0.6.2": + version: 0.6.2 + resolution: "webpack-virtual-modules@npm:0.6.2" + checksum: 10c0/5ffbddf0e84bf1562ff86cf6fcf039c74edf09d78358a6904a09bbd4484e8bb6812dc385fe14330b715031892dcd8423f7a88278b57c9f5002c84c2860179add + languageName: node + linkType: hard + "whatwg-encoding@npm:^2.0.0": version: 2.0.0 resolution: "whatwg-encoding@npm:2.0.0" @@ -20890,6 +21960,21 @@ __metadata: languageName: node linkType: hard +"yargs@npm:^17.5.1": + version: 17.7.3 + resolution: "yargs@npm:17.7.3" + dependencies: + cliui: "npm:^8.0.1" + escalade: "npm:^3.1.1" + get-caller-file: "npm:^2.0.5" + require-directory: "npm:^2.1.1" + string-width: "npm:^4.2.3" + y18n: "npm:^5.0.5" + yargs-parser: "npm:^21.1.1" + checksum: 10c0/7a28572f7e785a57886e34fdbddb9b28756dec552e1453d5f6e7cdd00ad8721a4e8c4321d33683f5e61cacb36ad43258adbb48396b71ec4ed14abee0fc0d0c1f + languageName: node + linkType: hard + "yargs@npm:^17.6.2, yargs@npm:^17.7.2": version: 17.7.2 resolution: "yargs@npm:17.7.2" From 4a538919d16598d70b5e546bb2b222aa6ed47d45 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:06:26 +0200 Subject: [PATCH 06/19] vision-camera-source: author the WebGPU shaders in TypeGPU (TGSL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the /webgpu passthrough shader from hand-written WGSL strings to TypeGPU: - FrameCropParams is now a d.struct — one source of truth for the WGSL struct decl and the 40-byte worklet packer (size derived via d.sizeOf). - vertexMain / fragmentMain are TGSL functions (tgpu.vertexFn/fragmentFn); the crop uniform is a tgpu.bindGroupLayout. - sampleCamera is a WGSL-bodied tgpu.fn the TGSL fragment calls. It stays WGSL because typegpu 0.11 can't resolve a texture_external inside a TGSL function; its texture+sampler bindings are declared as raw WGSL too. This is the only part not authored in TGSL. - Public API now exposes the sampleCamera TgpuFn + bindingDeclarations instead of a prebuilt shaderCode string (breaking). The per-frame worklet path stays raw react-native-webgpu — TGSL resolves to WGSL only at pipeline setup, so no tgpu proxy runs per frame. Verified: generated WGSL is functionally identical to the previous hand-written shader across ios/android x mirror (Android YUV decode byte-identical; only cosmetic naming + an intentional bind-group index swap differ), so GPU work is unchanged. CPU packing is untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../README.md | 43 ++++-- .../src/webgpu.ts | 9 +- .../src/webgpu/cameraPassthroughPipeline.ts | 123 +++++++++++------- .../src/webgpu/cameraShaderBindings.ts | 81 ++++++++---- .../src/webgpu/cropUtilities.ts | 32 +++-- 5 files changed, 190 insertions(+), 98 deletions(-) diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md index 6d028931..d0848b85 100644 --- a/packages/react-native-vision-camera-source/README.md +++ b/packages/react-native-vision-camera-source/README.md @@ -61,7 +61,13 @@ The `/webgpu` entry draws your own content — shaders, overlays, effects — in video with zero pixel copies. The hook owns the output surfaces, GPU synchronization with the encoder, timestamps, and frame lifetimes; your worklet only encodes passes. +Shaders are authored in [TypeGPU](https://docs.swmansion.com/TypeGPU/) (TGSL) — typed functions +that compile to WGSL. Enable the transform by adding `unplugin-typegpu` to your app's Babel config. + ```tsx +import tgpu from 'typegpu'; +import * as d from 'typegpu/data'; +import { dot } from 'typegpu/std'; import { useVisionCameraWebGpuSource, useCameraWebGpuDevice, @@ -70,24 +76,34 @@ import { type WebGpuFrameRenderFunction, } from '@fishjam-cloud/react-native-vision-camera-source/webgpu'; +// Full-screen triangle; uv spans the visible area. +const vertexMain = tgpu['~unstable'].vertexFn({ + in: { vertexIndex: d.builtin.vertexIndex }, + out: { position: d.builtin.position, uv: d.location(0, d.vec2f) }, +})((input) => { + const positions = [d.vec2f(-1, -1), d.vec2f(3, -1), d.vec2f(-1, 3)]; + const p = positions[input.vertexIndex]; + return { position: d.vec4f(p.x, p.y, 0, 1), uv: d.vec2f((p.x + 1) * 0.5, 1 - (p.y + 1) * 0.5) }; +}); + const { device } = useCameraWebGpuDevice(); const effect = useMemo(() => { if (device == null) return null; const cameraBindings = createCameraShaderBindings(device); + // Call cameraBindings.sampleCamera(uv) from your fragment — the platform's YUV decode is handled. + const fragmentMain = tgpu['~unstable'].fragmentFn({ in: { uv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { + const color = cameraBindings.sampleCamera(input.uv); + const gray = dot(color.xyz, d.vec3f(0.299, 0.587, 0.114)); // grayscale + return d.vec4f(gray, gray, gray, 1); + }); + // TypeGPU can't emit the external-texture binding, so prepend cameraBindings.bindingDeclarations. + const module = device.createShaderModule({ + code: cameraBindings.bindingDeclarations + tgpu.resolve({ externals: { vertexMain, fragmentMain } }), + }); const pipeline = device.createRenderPipeline({ layout: device.createPipelineLayout({ bindGroupLayouts: [cameraBindings.bindGroupLayout] }), - vertex: { module: device.createShaderModule({ code: FULL_SCREEN_TRIANGLE_WGSL }) }, - fragment: { - module: device.createShaderModule({ - code: - cameraBindings.shaderCode + - `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f { - let color = sampleCamera(uv); // upright RGB on BOTH platforms - return vec4f(vec3f(dot(color.rgb, vec3f(0.299, 0.587, 0.114))), 1.0); // grayscale - }`, - }), - targets: [{ format: getOutputSurfaceFormat() }], - }, + vertex: { module, entryPoint: 'vertexMain' }, + fragment: { module, entryPoint: 'fragmentMain', targets: [{ format: getOutputSurfaceFormat() }] }, }); return { cameraBindings, pipeline }; }, [device]); @@ -129,4 +145,5 @@ into a plain texture with `createCameraTextureResolver`. ## Development This package is part of the [web-client-sdk](https://github.com/fishjam-cloud/web-client-sdk) -monorepo. `yarn build` compiles `src/` to `dist/` with `tsc`. +monorepo. `yarn build` compiles `src/` to `dist/` with `react-native-builder-bob` (Babel + +`unplugin-typegpu` for the TGSL shaders, `tsc` for type definitions). diff --git a/packages/react-native-vision-camera-source/src/webgpu.ts b/packages/react-native-vision-camera-source/src/webgpu.ts index f22984fa..7a99b2fc 100644 --- a/packages/react-native-vision-camera-source/src/webgpu.ts +++ b/packages/react-native-vision-camera-source/src/webgpu.ts @@ -27,13 +27,20 @@ export { createCameraBindGroup, createCameraShaderBindings, type CreateCameraShaderBindingsOptions, + sampleCamera, } from './webgpu/cameraShaderBindings'; export { type CameraTextureResolver, createCameraTextureResolver, resolveCameraTexture, } from './webgpu/cameraTextureResolver'; -export { computeAspectFillCrop, computeSquareCrop, type FrameCrop, packFrameCropParams } from './webgpu/cropUtilities'; +export { + computeAspectFillCrop, + computeSquareCrop, + type FrameCrop, + FrameCropParams, + packFrameCropParams, +} from './webgpu/cropUtilities'; export type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './webgpu/frameRenderContext'; export { assertWebGpuDeviceSupportsCameraImport, diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts index 52685a09..06c12740 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts @@ -1,53 +1,64 @@ import { GPUBufferUsage, GPUShaderStage } from 'react-native-webgpu'; +import tgpu from 'typegpu'; +import * as d from 'typegpu/data'; +import { add, div, mul, sub } from 'typegpu/std'; -import { type CameraShaderBindings, createCameraBindGroup, createCameraShaderBindings } from './cameraShaderBindings'; -import { type FrameCrop, packFrameCropParams } from './cropUtilities'; +import { + type CameraShaderBindings, + createCameraBindGroup, + createCameraShaderBindings, + sampleCamera, +} from './cameraShaderBindings'; +import { type FrameCrop, FrameCropParams, packFrameCropParams } from './cropUtilities'; import { getOutputSurfaceFormat } from './requiredFeatures'; -const CAMERA_BIND_GROUP_INDEX = 0; -const CROP_BIND_GROUP_INDEX = 1; +// Crop is a plain uniform, so it is authored in TGSL and TypeGPU assigns it @group(0). The camera +// is a `texture_external`, which TypeGPU cannot resolve in a shader, so its bindings are raw WGSL +// and go at @group(1) (see cameraShaderBindings). This is a group swap from the pre-TGSL shader +// (which had camera at 0, crop at 1) but is otherwise identical. +const CROP_BIND_GROUP_INDEX = 0; +const CAMERA_BIND_GROUP_INDEX = 1; -const FRAME_CROP_BUFFER_BYTES = 40; +const FRAME_CROP_BUFFER_BYTES = d.sizeOf(FrameCropParams); -// One oversized triangle covering the viewport; uv spans [0,1] top-left origin over the visible -// area. The fragment applies the FrameCropParams crop + orientation transform (same math as the -// charades composite) and samples the camera via the injected sampleCamera(). -function buildPassthroughShaderCode(cameraShaderCode: string, mirror: boolean): string { - const cropUvExpression = mirror ? 'vec2f(1.0 - screenUv.x, screenUv.y)' : 'screenUv'; - return /* wgsl */ `${cameraShaderCode} -struct FrameCropParams { - sourceSize: vec2u, - cropOrigin: vec2f, - cropSize: vec2f, - uvTransform: mat2x2f, -} - -@group(${CROP_BIND_GROUP_INDEX}) @binding(0) var cropParams: FrameCropParams; - -struct VertexOutput { - @builtin(position) position: vec4f, - @location(0) uv: vec2f, -} +// TypeGPU bind group layout for the crop uniform — the source of truth for the WGSL declaration +// and for the fragment's typed access to `cropParams`. +const cropLayout = tgpu.bindGroupLayout({ cropParams: { uniform: FrameCropParams } }); -@vertex -fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { - var positions = array(vec2f(-1.0, -1.0), vec2f(3.0, -1.0), vec2f(-1.0, 3.0)); - let position = positions[vertexIndex]; - var output: VertexOutput; - output.position = vec4f(position, 0.0, 1.0); - output.uv = vec2f((position.x + 1.0) * 0.5, 1.0 - (position.y + 1.0) * 0.5); - return output; -} +// One oversized triangle covering the viewport; uv spans [0,1] top-left origin over the visible +// area. +const vertexMain = tgpu['~unstable'] + .vertexFn({ + in: { vertexIndex: d.builtin.vertexIndex }, + out: { position: d.builtin.position, uv: d.location(0, d.vec2f) }, + })((input) => { + const positions = [d.vec2f(-1, -1), d.vec2f(3, -1), d.vec2f(-1, 3)]; + const position = positions[input.vertexIndex]; + return { + position: d.vec4f(position.x, position.y, 0, 1), + uv: d.vec2f((position.x + 1) * 0.5, 1 - (position.y + 1) * 0.5), + }; + }) + .$name('vertexMain'); -@fragment -fn fragmentMain(@location(0) screenUv: vec2f) -> @location(0) vec4f { - let cropUv = ${cropUvExpression}; - let sourcePixel = cropParams.cropOrigin + cropUv * cropParams.cropSize; - let sourceUv = sourcePixel / vec2f(cropParams.sourceSize); - let cameraUv = cropParams.uvTransform * (sourceUv - vec2f(0.5)) + vec2f(0.5); - return sampleCamera(cameraUv); -} -`; +// The fragment applies the FrameCropParams crop + orientation transform and samples the camera via +// sampleCamera(). Mirroring is folded into a build-time sign/offset on uv.x (mirror → 1 - x) so the +// shader stays branch-free. +function makeFragmentMain(mirror: boolean) { + const mirrorSign = mirror ? -1 : 1; + const mirrorOffset = mirror ? 1 : 0; + return tgpu['~unstable'] + .fragmentFn({ in: { screenUv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { + // cropUv = (mirrorSign * screenUv.x + mirrorOffset, screenUv.y), expressed with vector ops + // so mirrorSign/mirrorOffset fold to literals and the WGSL stays branch-free. + const cropUv = add(mul(input.screenUv, d.vec2f(mirrorSign, 1)), d.vec2f(mirrorOffset, 0)); + const cp = cropLayout.$.cropParams; + const sourcePixel = add(cp.cropOrigin, mul(cropUv, cp.cropSize)); + const sourceUv = div(sourcePixel, d.vec2f(cp.sourceSize)); + const cameraUv = add(mul(cp.uvTransform, sub(sourceUv, d.vec2f(0.5))), d.vec2f(0.5)); + return sampleCamera(cameraUv); + }) + .$name('fragmentMain'); } /** Options for {@link createCameraPassthroughPipeline}. */ @@ -66,14 +77,27 @@ export interface CameraPassthroughPipelineOptions { */ export interface CameraPassthroughPipeline { readonly pipeline: GPURenderPipeline; - /** The camera shader bindings the pipeline samples through (group 0). */ + /** The camera shader bindings the pipeline samples through. */ readonly cameraShaderBindings: CameraShaderBindings; /** Uniform buffer holding the packed FrameCropParams; written by {@link encodeCameraPassthrough}. */ readonly cropParamsBuffer: GPUBuffer; - /** Static bind group with the crop uniform (group 1). */ + /** Static bind group with the crop uniform. */ readonly cropBindGroup: GPUBindGroup; } +/** + * Resolves the passthrough shader to WGSL: the raw camera-texture declarations (which TypeGPU + * cannot emit) followed by the TGSL-authored vertex + fragment functions, `sampleCamera`, and the + * `FrameCropParams` struct + crop uniform. + */ +function buildPassthroughShaderCode(cameraShaderBindings: CameraShaderBindings, mirror: boolean): string { + const resolved = tgpu.resolve({ + externals: { vertexMain, fragmentMain: makeFragmentMain(mirror) }, + names: 'strict', + }); + return `${cameraShaderBindings.bindingDeclarations}\n${resolved}`; +} + /** * Builds the full-screen camera passthrough pipeline: crop/orientation via {@link FrameCrop}, * platform-correct camera sampling, one triangle. Use it to publish the camera through the WebGPU @@ -104,13 +128,14 @@ export function createCameraPassthroughPipeline( const shaderModule = device.createShaderModule({ label: 'fishjam-camera-passthrough', - code: buildPassthroughShaderCode(cameraShaderBindings.shaderCode, options.mirror ?? false), + code: buildPassthroughShaderCode(cameraShaderBindings, options.mirror ?? false), }); + const bindGroupLayouts: GPUBindGroupLayout[] = []; + bindGroupLayouts[CROP_BIND_GROUP_INDEX] = cropBindGroupLayout; + bindGroupLayouts[CAMERA_BIND_GROUP_INDEX] = cameraShaderBindings.bindGroupLayout; const pipeline = device.createRenderPipeline({ label: 'fishjam-camera-passthrough', - layout: device.createPipelineLayout({ - bindGroupLayouts: [cameraShaderBindings.bindGroupLayout, cropBindGroupLayout], - }), + layout: device.createPipelineLayout({ bindGroupLayouts }), vertex: { module: shaderModule, entryPoint: 'vertexMain' }, fragment: { module: shaderModule, @@ -145,8 +170,8 @@ export function encodeCameraPassthrough( colorAttachments: [{ view: outputView, loadOp: 'clear', storeOp: 'store', clearValue: [0, 0, 0, 1] }], }); pass.setPipeline(passthrough.pipeline); - pass.setBindGroup(CAMERA_BIND_GROUP_INDEX, cameraBindGroup); pass.setBindGroup(CROP_BIND_GROUP_INDEX, passthrough.cropBindGroup); + pass.setBindGroup(CAMERA_BIND_GROUP_INDEX, cameraBindGroup); pass.draw(3); pass.end(); } diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts index c3818f4e..f36b9017 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts @@ -1,11 +1,20 @@ import { Platform } from 'react-native'; import { GPUShaderStage } from 'react-native-webgpu'; +import tgpu, { type TgpuFn } from 'typegpu'; +import * as d from 'typegpu/data'; +// The camera arrives as a `texture_external`, which TypeGPU 0.11 cannot resolve inside a TGSL +// function (its type only resolves as a declaration, not as a sampled value in codegen). So +// `sampleCamera` is authored as a WGSL-bodied `tgpu.fn`: it is a first-class TypeGPU function that +// TGSL shaders can call, but its body is WGSL and it references the external texture + sampler by +// name — those bindings are declared separately by {@link CameraShaderBindings.bindingDeclarations} +// (also WGSL, for the same reason). This is the one part of the shader that is not authored in TGSL. +// // On Android, the camera arrives as an opaque YCbCr AHardwareBuffer and Dawn's Vulkan path forces // an identity sampler conversion, so sampling the external texture returns RAW [Y, Cb, Cr] — the // BT.709 limited-range decode below must run in-shader. iOS (NV12 IOSurface) samples as // ready-to-use RGB. -const SAMPLE_CAMERA_BODY_ANDROID = /* wgsl */ ` +const SAMPLE_CAMERA_ANDROID = /* wgsl */ `(uv: vec2f) -> vec4f { let rawSample = textureSampleBaseClampToEdge(fishjamCameraTexture, fishjamCameraSampler, uv); let luma = rawSample.r - 0.0627451; let chromaBlue = rawSample.g - 0.5; @@ -15,25 +24,31 @@ const SAMPLE_CAMERA_BODY_ANDROID = /* wgsl */ ` 1.164384 * luma - 0.213249 * chromaBlue - 0.532909 * chromaRed, 1.164384 * luma + 2.112402 * chromaBlue, ); - return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0);`; + return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0); +}`; -const SAMPLE_CAMERA_BODY_IOS = /* wgsl */ ` - return textureSampleBaseClampToEdge(fishjamCameraTexture, fishjamCameraSampler, uv);`; +const SAMPLE_CAMERA_IOS = /* wgsl */ `(uv: vec2f) -> vec4f { + return textureSampleBaseClampToEdge(fishjamCameraTexture, fishjamCameraSampler, uv); +}`; -function buildCameraShaderCode(bindGroupIndex: number): string { - const body = Platform.OS === 'android' ? SAMPLE_CAMERA_BODY_ANDROID : SAMPLE_CAMERA_BODY_IOS; - return /* wgsl */ ` -@group(${bindGroupIndex}) @binding(0) var fishjamCameraTexture: texture_external; -@group(${bindGroupIndex}) @binding(1) var fishjamCameraSampler: sampler; - -fn sampleCamera(uv: vec2f) -> vec4f {${body} -} -`; -} +/** + * Samples the live camera and returns upright RGB on both platforms (the Android in-shader BT.709 + * YUV decode is included automatically). A TypeGPU function you can call from your own TGSL + * fragment shaders. It reads the camera texture + sampler declared by + * {@link CameraShaderBindings.bindingDeclarations}, which you must prepend to the resolved shader. + * + * @group WebGPU + */ +export const sampleCamera: TgpuFn<(uv: d.Vec2f) => d.Vec4f> = tgpu + .fn( + [d.vec2f], + d.vec4f, + )(Platform.OS === 'android' ? SAMPLE_CAMERA_ANDROID : SAMPLE_CAMERA_IOS) + .$name('sampleCamera'); /** Options for {@link createCameraShaderBindings}. */ export interface CreateCameraShaderBindingsOptions { - /** Bind group index the camera bindings are declared at in {@link CameraShaderBindings.shaderCode}. Defaults to `0`. */ + /** Bind group index the camera texture + sampler are declared at. Defaults to `0`. */ bindGroupIndex?: number; } @@ -45,11 +60,15 @@ export interface CreateCameraShaderBindingsOptions { */ export interface CameraShaderBindings { /** - * WGSL to prepend to your fragment shader. It declares the camera texture + sampler at - * {@link bindGroupIndex} and defines `sampleCamera(uv: vec2f) -> vec4f`, which returns upright - * RGB on both platforms (the Android in-shader YUV decode is included automatically). + * The camera sampler as a TypeGPU function — call `sampleCamera(uv)` from your TGSL fragment + * shader. Same value as the exported {@link sampleCamera}. + */ + readonly sampleCamera: typeof sampleCamera; + /** + * WGSL declaring the camera `texture_external` and `sampler` at {@link bindGroupIndex}. TypeGPU + * cannot emit an external-texture binding, so prepend this to the WGSL your shader resolves to. */ - readonly shaderCode: string; + readonly bindingDeclarations: string; /** Layout of the camera bind group; place it at {@link bindGroupIndex} in your pipeline layout. */ readonly bindGroupLayout: GPUBindGroupLayout; /** The linear-filtering sampler bound at binding 1. */ @@ -58,15 +77,24 @@ export interface CameraShaderBindings { readonly bindGroupIndex: number; } +function buildBindingDeclarations(bindGroupIndex: number): string { + return /* wgsl */ `@group(${bindGroupIndex}) @binding(0) var fishjamCameraTexture: texture_external; +@group(${bindGroupIndex}) @binding(1) var fishjamCameraSampler: sampler; +`; +} + /** - * Builds the camera-sampling shader bindings against the device your pipelines use. Pass the - * result to the source hook's `cameraShaderBindings` option and the render context delivers a - * ready-made `cameraBindGroup` every frame — set it at {@link CameraShaderBindings.bindGroupIndex} - * and call `sampleCamera(uv)` in your fragment shader. + * Builds the camera-sampling bindings against the device your pipelines use. Pass the result to + * the source hook's `cameraShaderBindings` option and the render context delivers a ready-made + * `cameraBindGroup` every frame — set it at {@link CameraShaderBindings.bindGroupIndex} and call + * `sampleCamera(uv)` in your fragment shader. * * ```ts - * const cameraBindings = createCameraShaderBindings(device); - * const module = device.createShaderModule({ code: cameraBindings.shaderCode + MY_FRAGMENT_WGSL }); + * const cam = createCameraShaderBindings(device); + * const fragment = tgpu.fragmentFn({ in: { uv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { + * return cam.sampleCamera(input.uv); + * }); + * const wgsl = cam.bindingDeclarations + tgpu.resolve({ externals: { fragment } }); * ``` * * @group WebGPU @@ -85,7 +113,8 @@ export function createCameraShaderBindings( }); const sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' }); return { - shaderCode: buildCameraShaderCode(bindGroupIndex), + sampleCamera, + bindingDeclarations: buildBindingDeclarations(bindGroupIndex), bindGroupLayout, sampler, bindGroupIndex, diff --git a/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts b/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts index 67d1eb08..4b6c22eb 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts @@ -1,15 +1,25 @@ /** - * Crop math + a worklet-safe byte packer for the FrameCropParams uniform used by + * Crop math + a worklet-safe byte packer for the {@link FrameCropParams} uniform used by * {@link createCameraPassthroughPipeline} (and reusable by your own shaders). + */ + +import * as d from 'typegpu/data'; + +/** + * The TypeGPU schema for the crop uniform. This is the single source of truth for both the WGSL + * `struct FrameCropParams` (emitted when the passthrough shader is resolved) and the byte layout + * {@link packFrameCropParams} writes. Its std140 layout is 40 bytes: * - * FrameCropParams byte layout (40 bytes), matching the WGSL struct - * `{ sourceSize: vec2u, cropOrigin: vec2f, cropSize: vec2f, uvTransform: mat2x2f }`: + * sourceSize: vec2u @0 cropOrigin: vec2f @8 cropSize: vec2f @16 uvTransform: mat2x2f @24 * - * [0] u32 sourceSize.x [1] u32 sourceSize.y - * [2] f32 cropOrigin.x [3] f32 cropOrigin.y - * [4] f32 cropSize.x [5] f32 cropSize.y - * [6..9] f32 uvTransform, column-major (m00, m01, m10, m11) + * @group WebGPU */ +export const FrameCropParams = d.struct({ + sourceSize: d.vec2u, + cropOrigin: d.vec2f, + cropSize: d.vec2f, + uvTransform: d.mat2x2f, +}); /** * A center-crop of a source frame plus a UV-space orientation transform, in source pixels. @@ -31,10 +41,14 @@ export interface FrameCrop { uv11: number; } -const FRAME_CROP_BYTES = 40; +// Derived from the schema so the buffer size can never drift from the WGSL struct. The manual +// offsets below match the schema's std140 layout (asserted in tests); we keep a hand-written +// packer rather than a schema-driven writer because this runs per-frame inside a worklet and must +// stay off the tgpu root/proxy. +const FRAME_CROP_BYTES = d.sizeOf(FrameCropParams); /** - * Packs a {@link FrameCrop} into the 40-byte FrameCropParams uniform layout (see the module doc). + * Packs a {@link FrameCrop} into the {@link FrameCropParams} uniform byte layout. * Worklet-safe; upload the result with `device.queue.writeBuffer(buffer, 0, bytes)`. * * @group WebGPU From 0b452d0ab5d1f972aac6459bcc64b8f82131e1bb Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:17:14 +0200 Subject: [PATCH 07/19] vision-camera-source: use stable tgpu.vertexFn/fragmentFn tgpu['~unstable'].vertexFn/fragmentFn are @deprecated aliases (the feature graduated to stable); switch to the top-level tgpu.vertexFn/tgpu.fragmentFn. Same function reference, so the generated WGSL is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/react-native-vision-camera-source/README.md | 4 ++-- .../src/webgpu/cameraPassthroughPipeline.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md index d0848b85..64c00143 100644 --- a/packages/react-native-vision-camera-source/README.md +++ b/packages/react-native-vision-camera-source/README.md @@ -77,7 +77,7 @@ import { } from '@fishjam-cloud/react-native-vision-camera-source/webgpu'; // Full-screen triangle; uv spans the visible area. -const vertexMain = tgpu['~unstable'].vertexFn({ +const vertexMain = tgpu.vertexFn({ in: { vertexIndex: d.builtin.vertexIndex }, out: { position: d.builtin.position, uv: d.location(0, d.vec2f) }, })((input) => { @@ -91,7 +91,7 @@ const effect = useMemo(() => { if (device == null) return null; const cameraBindings = createCameraShaderBindings(device); // Call cameraBindings.sampleCamera(uv) from your fragment — the platform's YUV decode is handled. - const fragmentMain = tgpu['~unstable'].fragmentFn({ in: { uv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { + const fragmentMain = tgpu.fragmentFn({ in: { uv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { const color = cameraBindings.sampleCamera(input.uv); const gray = dot(color.xyz, d.vec3f(0.299, 0.587, 0.114)); // grayscale return d.vec4f(gray, gray, gray, 1); diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts index 06c12740..09dd5a8b 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts @@ -27,7 +27,7 @@ const cropLayout = tgpu.bindGroupLayout({ cropParams: { uniform: FrameCropParams // One oversized triangle covering the viewport; uv spans [0,1] top-left origin over the visible // area. -const vertexMain = tgpu['~unstable'] +const vertexMain = tgpu .vertexFn({ in: { vertexIndex: d.builtin.vertexIndex }, out: { position: d.builtin.position, uv: d.location(0, d.vec2f) }, @@ -47,7 +47,7 @@ const vertexMain = tgpu['~unstable'] function makeFragmentMain(mirror: boolean) { const mirrorSign = mirror ? -1 : 1; const mirrorOffset = mirror ? 1 : 0; - return tgpu['~unstable'] + return tgpu .fragmentFn({ in: { screenUv: d.location(0, d.vec2f) }, out: d.vec4f })((input) => { // cropUv = (mirrorSign * screenUv.x + mirrorOffset, screenUv.y), expressed with vector ops // so mirrorSign/mirrorOffset fold to literals and the WGSL stays branch-free. From be75b5fabf722e47cb1181fcfcb5b06f98d90cc3 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:24:47 +0200 Subject: [PATCH 08/19] vision-camera-source: document /webgpu tier requirements (iOS 17, babel) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spell out that the WebGPU entry needs react-native-webgpu, the unplugin-typegpu Babel plugin (for the TGSL shaders), and iOS 17+ for the Metal camera-import path — and that the base camera-publishing tier requires none of these. The package sets no deployment target itself (pure JS); the app chooses its own. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/react-native-vision-camera-source/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md index 64c00143..80a4fcf9 100644 --- a/packages/react-native-vision-camera-source/README.md +++ b/packages/react-native-vision-camera-source/README.md @@ -15,7 +15,13 @@ component stays fully declarative. plugin configured (required by VisionCamera's frame outputs) - `@fishjam-cloud/react-native-client` with your app wrapped in `FishjamProvider` - New Architecture (custom video tracks require it) -- For the `/webgpu` entry: `react-native-webgpu` ≥ 0.5.15 (optional otherwise) +- For the `/webgpu` entry only (the base camera-publishing tier needs none of these): + - `react-native-webgpu` ≥ 0.5.15 + - `unplugin-typegpu` in your app's Babel config — the WebGPU shaders are authored in + [TypeGPU](https://docs.swmansion.com/TypeGPU/) (TGSL) and need its build-time transform + - **iOS 17+** recommended: the camera-import path relies on Metal external-texture features not + guaranteed on earlier versions. The base tier has no such requirement — set your app's + `ios.deploymentTarget` to whatever the base tier supports and gate WebGPU usage accordingly. ## Publish the camera From 0f9e4a4ca03107e877936ca633fe5a8dd43ada30 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:27:26 +0200 Subject: [PATCH 09/19] Revert iOS deployment target bump in withFishjamIos plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR had bumped the mobile-client Expo plugin's default IPHONEOS_DEPLOYMENT_TARGET from 15.1 to 17.0, which forces iOS 17 on every @fishjam-cloud/react-native-client consumer — not just those using the WebGPU tier. Restore 15.1; apps that need a higher floor (e.g. for the /webgpu camera-import path) can still set it via options.ios.iphoneDeploymentTarget. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mobile-client/plugin/src/withFishjamIos.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mobile-client/plugin/src/withFishjamIos.ts b/packages/mobile-client/plugin/src/withFishjamIos.ts index 83f54626..dab92eee 100644 --- a/packages/mobile-client/plugin/src/withFishjamIos.ts +++ b/packages/mobile-client/plugin/src/withFishjamIos.ts @@ -14,7 +14,7 @@ function getSbeDisplayName(props: FishjamPluginOptions) { } const TARGETED_DEVICE_FAMILY = `"1,2"`; -const IPHONEOS_DEPLOYMENT_TARGET = '17.0'; +const IPHONEOS_DEPLOYMENT_TARGET = '15.1'; const GROUP_IDENTIFIER_TEMPLATE_REGEX = /{{GROUP_IDENTIFIER}}/gm; const BUNDLE_IDENTIFIER_TEMPLATE_REGEX = /{{BUNDLE_IDENTIFIER}}/gm; const DISPLAY_NAME_TEMPLATE_REGEX = /{{DISPLAY_NAME}}/gm; From a6c7e39ee81865ff1df4e74629ee52becc215949 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:30:08 +0200 Subject: [PATCH 10/19] vision-camera-source: warn on unsupported iOS for the /webgpu tier Add warnIfIosVersionUnsupported(): a one-time console.warn when the /webgpu camera tier runs on iOS < 17, where Metal external-texture camera import may fail. Surfaces a clear, actionable reason before the cryptic device/feature error. Called automatically by useCameraWebGpuDevice (the choke point both shared and bring-your-own-device paths pass through) and exported for advanced callers that skip the hook. No-op off iOS. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/webgpu.ts | 1 + .../src/webgpu/requiredFeatures.ts | 32 +++++++++++++++++++ .../src/webgpu/useCameraWebGpuDevice.ts | 7 +++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/react-native-vision-camera-source/src/webgpu.ts b/packages/react-native-vision-camera-source/src/webgpu.ts index 7a99b2fc..e2e270ec 100644 --- a/packages/react-native-vision-camera-source/src/webgpu.ts +++ b/packages/react-native-vision-camera-source/src/webgpu.ts @@ -46,6 +46,7 @@ export { assertWebGpuDeviceSupportsCameraImport, getOutputSurfaceFormat, getRequiredWebGpuCameraFeatures, + warnIfIosVersionUnsupported, } from './webgpu/requiredFeatures'; export { useCameraWebGpuDevice, type UseCameraWebGpuDeviceResult } from './webgpu/useCameraWebGpuDevice'; export { diff --git a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts index f56288ba..6db761d0 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts @@ -41,6 +41,38 @@ export function assertWebGpuDeviceSupportsCameraImport(device: GPUDevice): void ); } +// The camera-import Metal features (external textures, multi-planar formats) aren't guaranteed +// before this iOS major version, so the /webgpu tier can fail on older devices even though it +// compiles. Best-effort heuristic; the real gate is assertWebGpuDeviceSupportsCameraImport. +const MIN_IOS_MAJOR_FOR_CAMERA_IMPORT = 17; +let hasWarnedUnsupportedIos = false; + +/** + * Emits a one-time console warning when running on an iOS version below the floor the camera-import + * path needs (currently iOS {@link MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}). On older versions device + * acquisition or per-frame camera import can fail; this surfaces a clear reason before the more + * cryptic device/feature error. No-op on Android and other platforms. {@link useCameraWebGpuDevice} + * calls this for you. + * + * @group WebGPU + */ +export function warnIfIosVersionUnsupported(): void { + if (hasWarnedUnsupportedIos || Platform.OS !== 'ios') { + return; + } + const iosMajor = parseInt(String(Platform.Version), 10); + if (Number.isNaN(iosMajor) || iosMajor >= MIN_IOS_MAJOR_FOR_CAMERA_IMPORT) { + return; + } + hasWarnedUnsupportedIos = true; + console.warn( + `[react-native-vision-camera-source] The /webgpu camera tier needs iOS ${MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}+ ` + + `for Metal external-texture camera import, but this device runs iOS ${Platform.Version}. Camera import ` + + `may fail here — use the base useVisionCameraSource tier, or gate /webgpu usage to iOS ` + + `${MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}+.`, + ); +} + /** * The pixel format of Fishjam output surfaces on this platform: `'rgba8unorm'` on Android * (AHardwareBuffer), `'bgra8unorm'` on iOS (IOSurface). Use it as the render-target format of any diff --git a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts index 0178ea99..dd95fdbb 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts @@ -1,6 +1,10 @@ import { useEffect, useMemo, useState } from 'react'; -import { assertWebGpuDeviceSupportsCameraImport, getRequiredWebGpuCameraFeatures } from './requiredFeatures'; +import { + assertWebGpuDeviceSupportsCameraImport, + getRequiredWebGpuCameraFeatures, + warnIfIosVersionUnsupported, +} from './requiredFeatures'; import { getWebGpuRuntime } from './webGpuRuntime'; // One app-wide device: every hook instance shares it, so per-device resources (pipelines, @@ -62,6 +66,7 @@ export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult { const [result, setResult] = useState({ device: null, error: null }); useEffect(() => { + warnIfIosVersionUnsupported(); let cancelled = false; getSharedCameraWebGpuDevice() .then((device) => { From 9d6bf983f955b0987e81c27ceb4c1fa1000181a7 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:31:42 +0200 Subject: [PATCH 11/19] Revert "vision-camera-source: warn on unsupported iOS for the /webgpu tier" This reverts commit a6c7e39ee81865ff1df4e74629ee52becc215949. --- .../src/webgpu.ts | 1 - .../src/webgpu/requiredFeatures.ts | 32 ------------------- .../src/webgpu/useCameraWebGpuDevice.ts | 7 +--- 3 files changed, 1 insertion(+), 39 deletions(-) diff --git a/packages/react-native-vision-camera-source/src/webgpu.ts b/packages/react-native-vision-camera-source/src/webgpu.ts index e2e270ec..7a99b2fc 100644 --- a/packages/react-native-vision-camera-source/src/webgpu.ts +++ b/packages/react-native-vision-camera-source/src/webgpu.ts @@ -46,7 +46,6 @@ export { assertWebGpuDeviceSupportsCameraImport, getOutputSurfaceFormat, getRequiredWebGpuCameraFeatures, - warnIfIosVersionUnsupported, } from './webgpu/requiredFeatures'; export { useCameraWebGpuDevice, type UseCameraWebGpuDeviceResult } from './webgpu/useCameraWebGpuDevice'; export { diff --git a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts index 6db761d0..f56288ba 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts @@ -41,38 +41,6 @@ export function assertWebGpuDeviceSupportsCameraImport(device: GPUDevice): void ); } -// The camera-import Metal features (external textures, multi-planar formats) aren't guaranteed -// before this iOS major version, so the /webgpu tier can fail on older devices even though it -// compiles. Best-effort heuristic; the real gate is assertWebGpuDeviceSupportsCameraImport. -const MIN_IOS_MAJOR_FOR_CAMERA_IMPORT = 17; -let hasWarnedUnsupportedIos = false; - -/** - * Emits a one-time console warning when running on an iOS version below the floor the camera-import - * path needs (currently iOS {@link MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}). On older versions device - * acquisition or per-frame camera import can fail; this surfaces a clear reason before the more - * cryptic device/feature error. No-op on Android and other platforms. {@link useCameraWebGpuDevice} - * calls this for you. - * - * @group WebGPU - */ -export function warnIfIosVersionUnsupported(): void { - if (hasWarnedUnsupportedIos || Platform.OS !== 'ios') { - return; - } - const iosMajor = parseInt(String(Platform.Version), 10); - if (Number.isNaN(iosMajor) || iosMajor >= MIN_IOS_MAJOR_FOR_CAMERA_IMPORT) { - return; - } - hasWarnedUnsupportedIos = true; - console.warn( - `[react-native-vision-camera-source] The /webgpu camera tier needs iOS ${MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}+ ` + - `for Metal external-texture camera import, but this device runs iOS ${Platform.Version}. Camera import ` + - `may fail here — use the base useVisionCameraSource tier, or gate /webgpu usage to iOS ` + - `${MIN_IOS_MAJOR_FOR_CAMERA_IMPORT}+.`, - ); -} - /** * The pixel format of Fishjam output surfaces on this platform: `'rgba8unorm'` on Android * (AHardwareBuffer), `'bgra8unorm'` on iOS (IOSurface). Use it as the render-target format of any diff --git a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts index dd95fdbb..0178ea99 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts @@ -1,10 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; -import { - assertWebGpuDeviceSupportsCameraImport, - getRequiredWebGpuCameraFeatures, - warnIfIosVersionUnsupported, -} from './requiredFeatures'; +import { assertWebGpuDeviceSupportsCameraImport, getRequiredWebGpuCameraFeatures } from './requiredFeatures'; import { getWebGpuRuntime } from './webGpuRuntime'; // One app-wide device: every hook instance shares it, so per-device resources (pipelines, @@ -66,7 +62,6 @@ export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult { const [result, setResult] = useState({ device: null, error: null }); useEffect(() => { - warnIfIosVersionUnsupported(); let cancelled = false; getSharedCameraWebGpuDevice() .then((device) => { From 2a237baa2482ee9250d8f17ff9c5dee9e23dbac2 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 9 Jul 2026 16:35:06 +0200 Subject: [PATCH 12/19] Drop useCustomSourceManager.spec.ts Remove the custom-source manager test added for the setStream stability fix, per request. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/tests/useCustomSourceManager.spec.ts | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 packages/react-client/src/tests/useCustomSourceManager.spec.ts diff --git a/packages/react-client/src/tests/useCustomSourceManager.spec.ts b/packages/react-client/src/tests/useCustomSourceManager.spec.ts deleted file mode 100644 index 5161bb2b..00000000 --- a/packages/react-client/src/tests/useCustomSourceManager.spec.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { FishjamClient, Logger } from "@fishjam-cloud/ts-client"; -import { act, renderHook, waitFor } from "@testing-library/react"; -import { FakeMediaStreamTrack } from "fake-mediastreamtrack"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { useCustomSourceManager } from "../hooks/internal/useCustomSourceManager"; -import type { PeerStatus } from "../types/public"; - -function makeStream({ video = true, audio = false } = {}): MediaStream { - const videoTracks = video ? [new FakeMediaStreamTrack({ kind: "video" })] : []; - const audioTracks = audio ? [new FakeMediaStreamTrack({ kind: "audio" })] : []; - return { - getVideoTracks: () => videoTracks, - getAudioTracks: () => audioTracks, - } as unknown as MediaStream; -} - -function makeFishjamClient() { - return { - getLocalPeer: () => ({ metadata: { peer: { displayName: "tester" } } }), - addTrack: vi.fn(async () => "video-id"), - removeTrack: vi.fn(async () => undefined), - on: vi.fn(), - off: vi.fn(), - } as unknown as FishjamClient; -} - -const logger = { warn: vi.fn(), error: vi.fn(), info: vi.fn() } as unknown as Logger; - -describe("useCustomSourceManager", () => { - beforeEach(() => vi.clearAllMocks()); - - it("keeps setStream referentially stable across renders", () => { - const fishjamClient = makeFishjamClient(); - const { result, rerender } = renderHook( - ({ peerStatus }: { peerStatus: PeerStatus }) => useCustomSourceManager({ fishjamClient, peerStatus, logger }), - { initialProps: { peerStatus: "connecting" } }, - ); - - const first = result.current.setStream; - rerender({ peerStatus: "connected" }); - expect(result.current.setStream).toBe(first); - }); - - it("keeps setStream referentially stable after publishing mutates internal state", async () => { - const fishjamClient = makeFishjamClient(); - const { result } = renderHook(() => - // "connecting" so the source stays pending and no async publish effect runs — this isolates - // the referential-stability property from the publish lifecycle. - useCustomSourceManager({ fishjamClient, peerStatus: "connecting", logger }), - ); - - const before = result.current.setStream; - const stream = makeStream(); - await act(async () => { - await result.current.setStream("src-1", stream); - }); - - // Regression guard: setStream used to depend on the `sources` state, so calling it (which - // updates that state) handed back a brand-new function every time. - expect(result.current.setStream).toBe(before); - expect(result.current.getSource("src-1")?.stream).toBe(stream); - }); - - it("publishes a customVideo track when connected and removes it on setStream(null)", async () => { - const fishjamClient = makeFishjamClient(); - const { result } = renderHook(() => useCustomSourceManager({ fishjamClient, peerStatus: "connected", logger })); - - const stream = makeStream(); - await act(async () => { - await result.current.setStream("src-1", stream); - }); - - await waitFor(() => expect(fishjamClient.addTrack).toHaveBeenCalledTimes(1)); - const [publishedTrack, publishedMetadata] = vi.mocked(fishjamClient.addTrack).mock.calls[0]; - expect(publishedTrack).toBe(stream.getVideoTracks()[0]); - expect(publishedMetadata).toEqual({ type: "customVideo", displayName: "tester", paused: false }); - - await act(async () => { - await result.current.setStream("src-1", null); - }); - - expect(fishjamClient.removeTrack).toHaveBeenCalledWith("video-id"); - expect(result.current.getSource("src-1")).toBeUndefined(); - }); -}); From 64e6829aaabe90aa47dd3166efed9447b09de44b Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 11:33:52 +0200 Subject: [PATCH 13/19] vision-camera-source: untangle useManagedPooledTrack lifecycle The effect mixed a (async()=>{})().catch() IIFE, mutable pool/created vars reassigned in three places, and cancellation re-checked after every await. Extract the two-step allocation into allocatePooledTrack() (which owns the "free the orphaned pool if the track fails" case), collapse the effect to a single disposed flag + one allocation holder disposed exactly once, and pull out named helpers (toWorkletBufferDescriptor, toError, disposePool, disposeAllocation). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/internal/useManagedPooledTrack.ts | 107 +++++++++++------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts index ba0ea3ab..0d944768 100644 --- a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts +++ b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts @@ -1,8 +1,8 @@ import { createCustomVideoBufferPool, createCustomVideoTrack, + type CustomVideoBuffer, type CustomVideoBufferPool, - type CustomVideoTrackResult, type MediaStream, type PooledTrack, } from '@fishjam-cloud/react-native-webrtc'; @@ -26,16 +26,48 @@ interface ManagedPooledTrack { const INITIAL_STATE: ManagedPooledTrack = { track: null, stream: null, bufferDescriptors: null, error: null }; -function disposeCreated(pool: CustomVideoBufferPool | null, created: CustomVideoTrackResult | null): void { - // Stop the track first, then free the pool — the pool must outlive the track's last frame. +/** A fully-allocated pool + its track, owned and torn down as a single unit. */ +interface PooledTrackAllocation { + pool: CustomVideoBufferPool; + track: PooledTrack; + stream: MediaStream; + bufferDescriptors: WorkletBufferDescriptor[]; +} + +function toWorkletBufferDescriptor(buffer: CustomVideoBuffer): WorkletBufferDescriptor { + return { index: buffer.index, surfaceHandle: buffer.surfaceHandle, width: buffer.width, height: buffer.height }; +} + +function toError(cause: unknown): Error { + return cause instanceof Error ? cause : new Error(String(cause)); +} + +function disposePool(pool: CustomVideoBufferPool): void { + void pool.dispose().catch((cause: unknown) => { + console.warn('useManagedPooledTrack: disposing the buffer pool failed', cause); + }); +} + +/** Tears an allocation down in the required order: stop the track's frames, then free its pool. */ +function disposeAllocation({ pool, stream }: PooledTrackAllocation): void { try { - created?.stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); + stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); } catch (cause) { console.warn('useManagedPooledTrack: stopping tracks failed', cause); } - void pool?.dispose().catch((cause: unknown) => { - console.warn('useManagedPooledTrack: disposing the buffer pool failed', cause); - }); + disposePool(pool); +} + +/** Allocates the surface pool and a track bound to it. If the track fails, frees the orphan pool. */ +async function allocatePooledTrack(width: number, height: number, poolSize: number): Promise { + const pool = await createCustomVideoBufferPool({ width, height, poolSize }); + try { + const { track, stream } = await createCustomVideoTrack({ pool }); + return { pool, track, stream, bufferDescriptors: pool.buffers.map(toWorkletBufferDescriptor) }; + } catch (cause) { + disposePool(pool); + throw cause; + } } /** @@ -57,45 +89,38 @@ export function useManagedPooledTrack( if (!enabled) { return; } - let cancelled = false; - let pool: CustomVideoBufferPool | null = null; - let created: CustomVideoTrackResult | null = null; - (async () => { - pool = await createCustomVideoBufferPool({ width, height, poolSize }); - if (cancelled) { - disposeCreated(pool, null); - return; - } - created = await createCustomVideoTrack({ pool }); - if (cancelled) { - disposeCreated(pool, created); - return; - } - const bufferDescriptors = pool.buffers.map((buffer) => ({ - index: buffer.index, - surfaceHandle: buffer.surfaceHandle, - width: buffer.width, - height: buffer.height, - })); - setManagedTrack({ track: created.track, stream: created.stream, bufferDescriptors, error: null }); - })().catch((cause: unknown) => { - if (!cancelled) { + // Allocation is async, so the effect may be torn down before it resolves. `disposed` records + // that; `allocation` holds the result once it exists so cleanup can free it exactly once. + let disposed = false; + let allocation: PooledTrackAllocation | null = null; + + allocatePooledTrack(width, height, poolSize) + .then((result) => { + if (disposed) { + // Torn down while allocating — throw the just-built resources away. + disposeAllocation(result); + return; + } + allocation = result; setManagedTrack({ - track: null, - stream: null, - bufferDescriptors: null, - error: cause instanceof Error ? cause : new Error(String(cause)), + track: result.track, + stream: result.stream, + bufferDescriptors: result.bufferDescriptors, + error: null, }); - } - disposeCreated(pool, created); - pool = null; - created = null; - }); + }) + .catch((cause: unknown) => { + if (!disposed) { + setManagedTrack({ ...INITIAL_STATE, error: toError(cause) }); + } + }); return () => { - cancelled = true; - disposeCreated(pool, created); + disposed = true; + if (allocation) { + disposeAllocation(allocation); + } setManagedTrack(INITIAL_STATE); }; }, [enabled, width, height, poolSize]); From 783618b4b1372d89def5cf63f955c0e720e1d370 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 11:35:53 +0200 Subject: [PATCH 14/19] vision-camera-source: share toError, align useManagedForwardTrack useManagedForwardTrack was already clean, but tidy it for consistency with the pooled sibling: guard `disposed` (was `cancelled`) and only store the created track when not torn down, mirroring useManagedPooledTrack. Extract the repeated `cause instanceof Error ? cause : new Error(String(cause))` into a shared internal toError() (4 call sites across the two managed-track hooks and useCameraWebGpuDevice). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/internal/toError.ts | 4 ++++ .../src/internal/useManagedForwardTrack.ts | 21 ++++++++++--------- .../src/internal/useManagedPooledTrack.ts | 6 ++---- .../src/webgpu/useCameraWebGpuDevice.ts | 5 +++-- 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 packages/react-native-vision-camera-source/src/internal/toError.ts diff --git a/packages/react-native-vision-camera-source/src/internal/toError.ts b/packages/react-native-vision-camera-source/src/internal/toError.ts new file mode 100644 index 00000000..24aa3b97 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/internal/toError.ts @@ -0,0 +1,4 @@ +/** Coerce an unknown thrown value into an Error, leaving real Errors untouched. */ +export function toError(cause: unknown): Error { + return cause instanceof Error ? cause : new Error(String(cause)); +} diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts b/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts index 72cde124..b338eede 100644 --- a/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts +++ b/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts @@ -6,6 +6,8 @@ import { } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useState } from 'react'; +import { toError } from './toError'; + interface ManagedForwardTrack { track: ForwardTrack | null; stream: MediaStream | null; @@ -35,30 +37,29 @@ export function useManagedForwardTrack(enabled: boolean): ManagedForwardTrack { if (!enabled) { return; } - let cancelled = false; + // Creation is async, so the effect may be torn down before it resolves. `disposed` records + // that; `created` holds the result once it exists so cleanup can stop it exactly once. + let disposed = false; let created: CustomVideoTrackResult | null = null; createCustomVideoTrack() .then((result) => { - created = result; - if (cancelled) { + if (disposed) { + // Torn down while creating — throw the just-built track away. stopStreamTracks(result.stream); return; } + created = result; setManagedTrack({ track: result.track, stream: result.stream, error: null }); }) .catch((cause: unknown) => { - if (!cancelled) { - setManagedTrack({ - track: null, - stream: null, - error: cause instanceof Error ? cause : new Error(String(cause)), - }); + if (!disposed) { + setManagedTrack({ ...INITIAL_STATE, error: toError(cause) }); } }); return () => { - cancelled = true; + disposed = true; if (created != null) { stopStreamTracks(created.stream); } diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts index 0d944768..217ae648 100644 --- a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts +++ b/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts @@ -8,6 +8,8 @@ import { } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useState } from 'react'; +import { toError } from './toError'; + /** One pooled output surface as plain values the frame worklet can capture and import itself. */ export interface WorkletBufferDescriptor { index: number; @@ -38,10 +40,6 @@ function toWorkletBufferDescriptor(buffer: CustomVideoBuffer): WorkletBufferDesc return { index: buffer.index, surfaceHandle: buffer.surfaceHandle, width: buffer.width, height: buffer.height }; } -function toError(cause: unknown): Error { - return cause instanceof Error ? cause : new Error(String(cause)); -} - function disposePool(pool: CustomVideoBufferPool): void { void pool.dispose().catch((cause: unknown) => { console.warn('useManagedPooledTrack: disposing the buffer pool failed', cause); diff --git a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts index 0178ea99..7a4cf875 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts @@ -1,5 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; +import { toError } from '../internal/toError'; import { assertWebGpuDeviceSupportsCameraImport, getRequiredWebGpuCameraFeatures } from './requiredFeatures'; import { getWebGpuRuntime } from './webGpuRuntime'; @@ -71,7 +72,7 @@ export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult { }) .catch((cause: unknown) => { if (!cancelled) { - setResult({ device: null, error: cause instanceof Error ? cause : new Error(String(cause)) }); + setResult({ device: null, error: toError(cause) }); } }); return () => { @@ -97,7 +98,7 @@ export function useCameraWebGpuDeviceWithOverride(override: GPUDevice | undefine assertWebGpuDeviceSupportsCameraImport(override); return null; } catch (cause) { - return cause instanceof Error ? cause : new Error(String(cause)); + return toError(cause); } }, [override]); From af94b45eda06278a667f80ee7cacf2fa85e10d71 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 11:56:00 +0200 Subject: [PATCH 15/19] Split generic custom-video-source out of vision-camera-source The vision-camera dep was used in only 3 files; the rest (the custom-video track lifecycle hooks + the WebGPU camera toolkit) is source-agnostic. Extract it into a new package @fishjam-cloud/react-native-custom-video-source that publishes your own frames to Fishjam from any source, and make the VisionCamera package a thin adapter on top of it. - new package: internal/{useManagedForwardTrack,useManagedPooledTrack,toError} + webgpu/* (camera shader bindings, passthrough, crop, render context, required features, device, texture resolver, runtime). Owns typegpu + unplugin-typegpu (the TGSL shaders live here now). - vision-camera-source keeps only the VC-coupled files (useVisionCameraSource, useVisionCameraWebGpuSource, frameTimestamp, orientation), depends on the generic package (workspace:*), and its /webgpu re-exports the generic toolkit. Dropped its now-unused typegpu/unplugin-typegpu deps + babel plugin. - register the new workspace in the root package.json. Both packages typecheck, build (bob), and lint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 1 + .../.eslintignore | 3 + .../.eslintrc | 13 +++ .../.prettierrc | 9 ++ .../README.md | 70 +++++++++++ .../babel.config.js | 16 +++ .../package.json | 109 ++++++++++++++++++ .../src/index.ts | 21 ++++ .../src/internal/toError.ts | 0 .../src/internal/useManagedForwardTrack.ts | 0 .../src/internal/useManagedPooledTrack.ts | 0 .../src/webgpu.ts | 54 +++++++++ .../src/webgpu/cameraPassthroughPipeline.ts | 0 .../src/webgpu/cameraShaderBindings.ts | 0 .../src/webgpu/cameraTextureResolver.ts | 0 .../src/webgpu/cropUtilities.ts | 0 .../src/webgpu/frameRenderContext.ts | 0 .../src/webgpu/requiredFeatures.ts | 0 .../src/webgpu/useCameraWebGpuDevice.ts | 0 .../src/webgpu/webGpuRuntime.ts | 0 .../tsconfig.build.json | 9 ++ .../tsconfig.json | 18 +++ .../typedoc.json | 6 + .../babel.config.js | 13 +-- .../package.json | 7 +- .../src/useVisionCameraSource.ts | 2 +- .../src/webgpu.ts | 43 +------ .../src/webgpu/useVisionCameraWebGpuSource.ts | 16 ++- yarn.lock | 33 +++++- 29 files changed, 384 insertions(+), 59 deletions(-) create mode 100644 packages/react-native-custom-video-source/.eslintignore create mode 100644 packages/react-native-custom-video-source/.eslintrc create mode 100644 packages/react-native-custom-video-source/.prettierrc create mode 100644 packages/react-native-custom-video-source/README.md create mode 100644 packages/react-native-custom-video-source/babel.config.js create mode 100644 packages/react-native-custom-video-source/package.json create mode 100644 packages/react-native-custom-video-source/src/index.ts rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/internal/toError.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/internal/useManagedForwardTrack.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/internal/useManagedPooledTrack.ts (100%) create mode 100644 packages/react-native-custom-video-source/src/webgpu.ts rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/cameraPassthroughPipeline.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/cameraShaderBindings.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/cameraTextureResolver.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/cropUtilities.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/frameRenderContext.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/requiredFeatures.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/useCameraWebGpuDevice.ts (100%) rename packages/{react-native-vision-camera-source => react-native-custom-video-source}/src/webgpu/webGpuRuntime.ts (100%) create mode 100644 packages/react-native-custom-video-source/tsconfig.build.json create mode 100644 packages/react-native-custom-video-source/tsconfig.json create mode 100644 packages/react-native-custom-video-source/typedoc.json diff --git a/package.json b/package.json index 8eb876af..574d3279 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "packages/react-client", "packages/webrtc-client", "packages/mobile-client", + "packages/react-native-custom-video-source", "packages/react-native-vision-camera-source", "packages/react-native-webrtc", "examples/react-client/*", diff --git a/packages/react-native-custom-video-source/.eslintignore b/packages/react-native-custom-video-source/.eslintignore new file mode 100644 index 00000000..28bd0deb --- /dev/null +++ b/packages/react-native-custom-video-source/.eslintignore @@ -0,0 +1,3 @@ +dist +build +coverage diff --git a/packages/react-native-custom-video-source/.eslintrc b/packages/react-native-custom-video-source/.eslintrc new file mode 100644 index 00000000..a97fd5e1 --- /dev/null +++ b/packages/react-native-custom-video-source/.eslintrc @@ -0,0 +1,13 @@ +{ + "extends": ["../../.eslintrc.js", "expo", "prettier"], + "plugins": ["prettier"], + "ignorePatterns": ["lib", "/dist/*"], + "rules": { + "prettier/prettier": "error" + }, + "settings": { + "import/resolver": { + "typescript": true + } + } +} diff --git a/packages/react-native-custom-video-source/.prettierrc b/packages/react-native-custom-video-source/.prettierrc new file mode 100644 index 00000000..3ce79d75 --- /dev/null +++ b/packages/react-native-custom-video-source/.prettierrc @@ -0,0 +1,9 @@ +{ + "bracketSameLine": true, + "printWidth": 120, + "quoteProps": "consistent", + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "all", + "useTabs": false +} diff --git a/packages/react-native-custom-video-source/README.md b/packages/react-native-custom-video-source/README.md new file mode 100644 index 00000000..f963b2e1 --- /dev/null +++ b/packages/react-native-custom-video-source/README.md @@ -0,0 +1,70 @@ +# `@fishjam-cloud/react-native-custom-video-source` + +Publish your own video frames to Fishjam from **any** frame source. This is the generic layer +under [`@fishjam-cloud/react-native-vision-camera-source`](../react-native-vision-camera-source) — +use that package if your source is [VisionCamera](https://react-native-vision-camera.com); use this +one directly for any other source (a native ML pipeline, a compositor, your own renderer). + +There are two modes, picked by **how you produce frames**: + +- **Forwarding** — you already have finished native buffers. `useManagedForwardTrack()` creates and + owns the track; hand each buffer pointer to `forwardFrame` from `@fishjam-cloud/react-native-webrtc`. +- **Render-target (pooled)** — you render the frames yourself. `useManagedPooledTrack()` allocates a + surface pool; draw into it and hand each frame back with `pushFrame`. The + [`/webgpu`](#webgpu-camera-toolkit) entry point provides a WebGPU camera-rendering toolkit for this + mode. + +Each hook owns the track's async lifecycle — creation, the published `stream`, and teardown — so you +only feed frames. Must be used under `FishjamProvider` from `@fishjam-cloud/react-native-client`. + +## Prerequisites + +- `@fishjam-cloud/react-native-webrtc` and `@fishjam-cloud/react-native-client` (with your app wrapped + in `FishjamProvider`) +- New Architecture (custom video tracks require it) +- For the `/webgpu` entry only: `react-native-webgpu` ≥ 0.5.15, plus `unplugin-typegpu` in your app's + Babel config (the shaders are authored in [TypeGPU](https://docs.swmansion.com/TypeGPU/)), and + **iOS 17+** for the Metal external-texture camera-import path. + +## Forward finished buffers + +```tsx +import { useManagedForwardTrack } from '@fishjam-cloud/react-native-custom-video-source'; +import { forwardFrame } from '@fishjam-cloud/react-native-webrtc'; +import { useCustomSource } from '@fishjam-cloud/react-native-client'; + +const { track, stream } = useManagedForwardTrack(/* enabled */ true); +// publish `stream` (e.g. via useCustomSource), then per frame: +forwardFrame(track, { nativeBuffer /* bigint CVPixelBufferRef / AHardwareBuffer* */ }); +``` + +## Render your own frames + +```tsx +import { useManagedPooledTrack } from '@fishjam-cloud/react-native-custom-video-source'; +import { pushFrame } from '@fishjam-cloud/react-native-webrtc'; + +const { track, bufferDescriptors } = useManagedPooledTrack(true, 720, 1280, /* poolSize */ 3); +// import bufferDescriptors[i].surfaceHandle into your GPU, render into slot i, then: +pushFrame(track, { bufferIndex: i, timestampNs }); +``` + +## WebGPU camera toolkit + +`@fishjam-cloud/react-native-custom-video-source/webgpu` samples the live camera and helps you render +into the pooled surfaces: + +- `createCameraShaderBindings` / `sampleCamera` — sample the camera from your own shaders, YUV decode + handled per platform +- `createCameraPassthroughPipeline` / `encodeCameraPassthrough` — a ready-made camera→output pass +- `useCameraWebGpuDevice` — the shared, camera-import-capable `GPUDevice` +- `computeAspectFillCrop` / `computeSquareCrop` / `packFrameCropParams` — crop + orientation helpers + +> Verify camera + WebGPU behavior on **physical devices** — the iOS Simulator cannot import the +> camera's YUV textures. + +## Development + +Part of the [web-client-sdk](https://github.com/fishjam-cloud/web-client-sdk) monorepo. `yarn build` +compiles `src/` to `dist/` with `react-native-builder-bob` (Babel + `unplugin-typegpu` for the TGSL +shaders, `tsc` for type definitions). diff --git a/packages/react-native-custom-video-source/babel.config.js b/packages/react-native-custom-video-source/babel.config.js new file mode 100644 index 00000000..eeaf9dd7 --- /dev/null +++ b/packages/react-native-custom-video-source/babel.config.js @@ -0,0 +1,16 @@ +// Build-time Babel config, used only by react-native-builder-bob when compiling this package's +// own source to `dist` (bob is configured with `configFile: true`, so it defers entirely to this +// config instead of its default presets). It is never shipped — the published tarball is `dist` +// only — so it does not affect how consumers' Metro transpiles the built output. +// +// - `@babel/preset-typescript` strips the TypeScript types. It preserves `'worklet'` string +// directives, which must survive into `dist` so the consumer app's react-native-worklets Babel +// plugin can pick them up. We deliberately do NOT run the worklets plugin here — that is the +// consumer's job at app-build time. +// - `unplugin-typegpu/babel` transforms TGSL (`tgpu.fn(... )(js => ...)`) function bodies into the +// tinyest AST that TypeGPU resolves to WGSL at runtime. Without it, TGSL functions throw on +// `tgpu.resolve`. +module.exports = { + presets: [['@babel/preset-typescript', { onlyRemoveTypeImports: true }]], + plugins: [['unplugin-typegpu/babel', { forceTgpuAlias: 'tgpu' }]], +}; diff --git a/packages/react-native-custom-video-source/package.json b/packages/react-native-custom-video-source/package.json new file mode 100644 index 00000000..ff93fb29 --- /dev/null +++ b/packages/react-native-custom-video-source/package.json @@ -0,0 +1,109 @@ +{ + "name": "@fishjam-cloud/react-native-custom-video-source", + "version": "0.28.3", + "description": "Publish your own video frames to Fishjam — forward native buffers or render into a WebGPU surface pool, from any frame source", + "license": "Apache-2.0", + "author": "Fishjam Team", + "main": "./dist/module/index.js", + "module": "./dist/module/index.js", + "types": "./dist/typescript/index.d.ts", + "react-native": "./dist/module/index.js", + "exports": { + ".": { + "types": "./dist/typescript/index.d.ts", + "react-native": "./dist/module/index.js", + "import": "./dist/module/index.js", + "default": "./dist/module/index.js" + }, + "./webgpu": { + "types": "./dist/typescript/webgpu.d.ts", + "react-native": "./dist/module/webgpu.js", + "import": "./dist/module/webgpu.js", + "default": "./dist/module/webgpu.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/**" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/fishjam-cloud/web-client-sdk.git", + "directory": "packages/react-native-custom-video-source" + }, + "homepage": "https://github.com/fishjam-cloud/web-client-sdk#readme", + "bugs": "https://github.com/fishjam-cloud/web-client-sdk/issues", + "keywords": [ + "webrtc", + "fishjam", + "webgpu", + "custom-video" + ], + "scripts": { + "tsc": "tsc --noEmit", + "build:webrtc-workspace": "yarn workspace @fishjam-cloud/react-native-webrtc run prepare", + "build": "yarn build:webrtc-workspace && bob build", + "lint": "eslint . --ext .ts,.tsx --fix", + "prepare": "yarn build:webrtc-workspace && bob build", + "format": "prettier --write . --ignore-path ./.eslintignore", + "check:webrtc-published": "node ../../release-automation/check-webrtc-published.mjs" + }, + "lint-staged": { + "*": [ + "yarn format" + ], + "*.{js,ts,tsx}": [ + "yarn lint" + ] + }, + "react-native-builder-bob": { + "source": "src", + "output": "dist", + "targets": [ + [ + "module", + { + "configFile": true + } + ], + [ + "typescript", + { + "project": "tsconfig.build.json", + "tsc": "../../node_modules/.bin/tsc" + } + ] + ] + }, + "peerDependencies": { + "@fishjam-cloud/react-native-webrtc": "workspace:*", + "react": "*", + "react-native": "*", + "react-native-webgpu": ">=0.5.15" + }, + "peerDependenciesMeta": { + "react-native-webgpu": { + "optional": true + } + }, + "dependencies": { + "typegpu": "^0.11.8" + }, + "devDependencies": { + "@babel/core": "^7.29.0", + "@babel/preset-typescript": "^7.27.0", + "@fishjam-cloud/react-native-webrtc": "workspace:*", + "@types/react": "19.2.14", + "@webgpu/types": "0.1.65", + "eslint-config-expo": "~9.2.0", + "eslint-import-resolver-typescript": "^3.10.1", + "eslint-plugin-prettier": "^5.5.1", + "react": "19.2.3", + "react-native": "0.85.3", + "react-native-builder-bob": "^0.18.2", + "react-native-webgpu": "0.5.15", + "typescript": "^5.8.3", + "unplugin-typegpu": "^0.11.5" + }, + "packageManager": "yarn@4.16.0" +} diff --git a/packages/react-native-custom-video-source/src/index.ts b/packages/react-native-custom-video-source/src/index.ts new file mode 100644 index 00000000..50aaa1c1 --- /dev/null +++ b/packages/react-native-custom-video-source/src/index.ts @@ -0,0 +1,21 @@ +/** + * Publish your own video frames to Fishjam. + * + * Source-lifecycle hooks that create a custom video track, publish it, and clean it up — you + * supply the frames from any source. Two modes, picked by how you produce frames: + * + * - {@link useManagedForwardTrack} — you already have finished native buffers (a camera, a native + * ML pipeline, a compositor); forward each buffer pointer with `forwardFrame` from + * `@fishjam-cloud/react-native-webrtc`. + * - {@link useManagedPooledTrack} — you render the frames yourself; allocate a surface pool, draw + * into it, and hand each frame back with `pushFrame`. The `@fishjam-cloud/react-native-custom-video-source/webgpu` + * entry point provides a WebGPU camera-rendering toolkit for this mode. + * + * For a ready-made VisionCamera integration on top of this, use + * `@fishjam-cloud/react-native-vision-camera-source`. + * + * @packageDocumentation + */ + +export { useManagedForwardTrack } from './internal/useManagedForwardTrack'; +export { useManagedPooledTrack, type WorkletBufferDescriptor } from './internal/useManagedPooledTrack'; diff --git a/packages/react-native-vision-camera-source/src/internal/toError.ts b/packages/react-native-custom-video-source/src/internal/toError.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/internal/toError.ts rename to packages/react-native-custom-video-source/src/internal/toError.ts diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/internal/useManagedForwardTrack.ts rename to packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts diff --git a/packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/internal/useManagedPooledTrack.ts rename to packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts diff --git a/packages/react-native-custom-video-source/src/webgpu.ts b/packages/react-native-custom-video-source/src/webgpu.ts new file mode 100644 index 00000000..32c0aa0e --- /dev/null +++ b/packages/react-native-custom-video-source/src/webgpu.ts @@ -0,0 +1,54 @@ +/// +/** + * WebGPU camera toolkit — sample the live camera and render your own content into the video you + * publish with {@link useManagedPooledTrack}. Requires `react-native-webgpu` (an optional peer of + * this package; only this entry point loads it). + * + * - {@link createCameraShaderBindings} — sample the live camera from your own shaders via + * `sampleCamera(uv)`, with the platform's YUV decode handled for you. + * - {@link createCameraPassthroughPipeline} / {@link encodeCameraPassthrough} — a ready-made + * camera→output pass to publish the camera with zero WGSL, or to build overlays on. + * - {@link createCameraTextureResolver} — opt-in plain-texture camera for pipelines that can't + * sample `texture_external`. + * - {@link useCameraWebGpuDevice} — the shared, camera-import-capable GPUDevice. + * + * @packageDocumentation + */ + +export { + type CameraPassthroughPipeline, + type CameraPassthroughPipelineOptions, + createCameraPassthroughPipeline, + encodeCameraPassthrough, +} from './webgpu/cameraPassthroughPipeline'; +export { + type CameraShaderBindings, + createCameraBindGroup, + createCameraShaderBindings, + type CreateCameraShaderBindingsOptions, + sampleCamera, +} from './webgpu/cameraShaderBindings'; +export { + type CameraTextureResolver, + createCameraTextureResolver, + resolveCameraTexture, +} from './webgpu/cameraTextureResolver'; +export { + computeAspectFillCrop, + computeSquareCrop, + type FrameCrop, + FrameCropParams, + packFrameCropParams, +} from './webgpu/cropUtilities'; +export type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './webgpu/frameRenderContext'; +export { + assertWebGpuDeviceSupportsCameraImport, + getOutputSurfaceFormat, + getRequiredWebGpuCameraFeatures, +} from './webgpu/requiredFeatures'; +export { + useCameraWebGpuDevice, + type UseCameraWebGpuDeviceResult, + useCameraWebGpuDeviceWithOverride, +} from './webgpu/useCameraWebGpuDevice'; +export { getWebGpuRuntime, type WebGpuRuntime } from './webgpu/webGpuRuntime'; diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts b/packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/cameraPassthroughPipeline.ts rename to packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts b/packages/react-native-custom-video-source/src/webgpu/cameraShaderBindings.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/cameraShaderBindings.ts rename to packages/react-native-custom-video-source/src/webgpu/cameraShaderBindings.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts b/packages/react-native-custom-video-source/src/webgpu/cameraTextureResolver.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/cameraTextureResolver.ts rename to packages/react-native-custom-video-source/src/webgpu/cameraTextureResolver.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts b/packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/cropUtilities.ts rename to packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts b/packages/react-native-custom-video-source/src/webgpu/frameRenderContext.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/frameRenderContext.ts rename to packages/react-native-custom-video-source/src/webgpu/frameRenderContext.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts b/packages/react-native-custom-video-source/src/webgpu/requiredFeatures.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/requiredFeatures.ts rename to packages/react-native-custom-video-source/src/webgpu/requiredFeatures.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/useCameraWebGpuDevice.ts rename to packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts diff --git a/packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts b/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts similarity index 100% rename from packages/react-native-vision-camera-source/src/webgpu/webGpuRuntime.ts rename to packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts diff --git a/packages/react-native-custom-video-source/tsconfig.build.json b/packages/react-native-custom-video-source/tsconfig.build.json new file mode 100644 index 00000000..dbe3ee21 --- /dev/null +++ b/packages/react-native-custom-video-source/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/packages/react-native-custom-video-source/tsconfig.json b/packages/react-native-custom-video-source/tsconfig.json new file mode 100644 index 00000000..75a4cfb2 --- /dev/null +++ b/packages/react-native-custom-video-source/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "allowJs": true, + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "dist/", + "skipLibCheck": true, + "strict": true, + "target": "ESNext", + "lib": ["es2020", "DOM", "DOM.Iterable"], + "jsx": "react-jsx" + }, + "include": ["src/**/*"] +} diff --git a/packages/react-native-custom-video-source/typedoc.json b/packages/react-native-custom-video-source/typedoc.json new file mode 100644 index 00000000..b14f5ae9 --- /dev/null +++ b/packages/react-native-custom-video-source/typedoc.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "includeVersion": true, + "entryPoints": ["src/index.ts", "src/webgpu.ts"], + "readme": "none" +} diff --git a/packages/react-native-vision-camera-source/babel.config.js b/packages/react-native-vision-camera-source/babel.config.js index eeaf9dd7..e6a3f6c5 100644 --- a/packages/react-native-vision-camera-source/babel.config.js +++ b/packages/react-native-vision-camera-source/babel.config.js @@ -3,14 +3,11 @@ // config instead of its default presets). It is never shipped — the published tarball is `dist` // only — so it does not affect how consumers' Metro transpiles the built output. // -// - `@babel/preset-typescript` strips the TypeScript types. It preserves `'worklet'` string -// directives, which must survive into `dist` so the consumer app's react-native-worklets Babel -// plugin can pick them up. We deliberately do NOT run the worklets plugin here — that is the -// consumer's job at app-build time. -// - `unplugin-typegpu/babel` transforms TGSL (`tgpu.fn(... )(js => ...)`) function bodies into the -// tinyest AST that TypeGPU resolves to WGSL at runtime. Without it, TGSL functions throw on -// `tgpu.resolve`. +// `@babel/preset-typescript` strips the TypeScript types. It preserves `'worklet'` string +// directives, which must survive into `dist` so the consumer app's react-native-worklets Babel +// plugin can pick them up. We deliberately do NOT run the worklets plugin here — that is the +// consumer's job at app-build time. (The TGSL shaders, which need unplugin-typegpu, live in +// @fishjam-cloud/react-native-custom-video-source, not here.) module.exports = { presets: [['@babel/preset-typescript', { onlyRemoveTypeImports: true }]], - plugins: [['unplugin-typegpu/babel', { forceTgpuAlias: 'tgpu' }]], }; diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json index f7a42396..e16011d0 100644 --- a/packages/react-native-vision-camera-source/package.json +++ b/packages/react-native-vision-camera-source/package.json @@ -90,8 +90,8 @@ } }, "dependencies": { - "react-native-worklets": ">=0.9.0", - "typegpu": "^0.11.8" + "@fishjam-cloud/react-native-custom-video-source": "workspace:*", + "react-native-worklets": ">=0.9.0" }, "devDependencies": { "@babel/core": "^7.29.0", @@ -110,8 +110,7 @@ "react-native-vision-camera": "5.0.10", "react-native-vision-camera-worklets": "5.0.10", "react-native-webgpu": "0.5.15", - "typescript": "^5.8.3", - "unplugin-typegpu": "^0.11.5" + "typescript": "^5.8.3" }, "packageManager": "yarn@4.16.0" } diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts index b5bd4246..c5384557 100644 --- a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -1,4 +1,5 @@ import { useCustomSource } from '@fishjam-cloud/react-native-client'; +import { useManagedForwardTrack } from '@fishjam-cloud/react-native-custom-video-source'; import { forwardFrame, type MediaStream } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useMemo } from 'react'; import { @@ -10,7 +11,6 @@ import { } from 'react-native-vision-camera'; import { createFrameTimestampState, normalizeFrameTimestampNanoseconds } from './frameTimestamp'; -import { useManagedForwardTrack } from './internal/useManagedForwardTrack'; import { rotationDegreesFromOrientation } from './orientation'; /** diff --git a/packages/react-native-vision-camera-source/src/webgpu.ts b/packages/react-native-vision-camera-source/src/webgpu.ts index 7a99b2fc..cb43c04e 100644 --- a/packages/react-native-vision-camera-source/src/webgpu.ts +++ b/packages/react-native-vision-camera-source/src/webgpu.ts @@ -6,50 +6,17 @@ * * - {@link useVisionCameraWebGpuSource} — the source hook: camera in, your WebGPU passes, * published video out. - * - {@link createCameraShaderBindings} — sample the live camera from your own shaders via - * `sampleCamera(uv)`, with the platform's YUV decode handled for you. - * - {@link createCameraPassthroughPipeline} / {@link encodeCameraPassthrough} — a ready-made - * camera→output pass to publish the camera with zero WGSL, or to build overlays on. - * - {@link createCameraTextureResolver} — opt-in plain-texture camera for pipelines that can't - * sample `texture_external`. + * - Re-exports the WebGPU camera toolkit from + * `@fishjam-cloud/react-native-custom-video-source/webgpu` — `createCameraShaderBindings` / + * `sampleCamera`, `createCameraPassthroughPipeline`, the cropping helpers, the shared + * camera-import device, and so on — so you can build your shaders alongside the hook. * * @packageDocumentation */ -export { - type CameraPassthroughPipeline, - type CameraPassthroughPipelineOptions, - createCameraPassthroughPipeline, - encodeCameraPassthrough, -} from './webgpu/cameraPassthroughPipeline'; -export { - type CameraShaderBindings, - createCameraBindGroup, - createCameraShaderBindings, - type CreateCameraShaderBindingsOptions, - sampleCamera, -} from './webgpu/cameraShaderBindings'; -export { - type CameraTextureResolver, - createCameraTextureResolver, - resolveCameraTexture, -} from './webgpu/cameraTextureResolver'; -export { - computeAspectFillCrop, - computeSquareCrop, - type FrameCrop, - FrameCropParams, - packFrameCropParams, -} from './webgpu/cropUtilities'; -export type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './webgpu/frameRenderContext'; -export { - assertWebGpuDeviceSupportsCameraImport, - getOutputSurfaceFormat, - getRequiredWebGpuCameraFeatures, -} from './webgpu/requiredFeatures'; -export { useCameraWebGpuDevice, type UseCameraWebGpuDeviceResult } from './webgpu/useCameraWebGpuDevice'; export { useVisionCameraWebGpuSource, type UseVisionCameraWebGpuSourceOptions, type UseVisionCameraWebGpuSourceResult, } from './webgpu/useVisionCameraWebGpuSource'; +export * from '@fishjam-cloud/react-native-custom-video-source/webgpu'; diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index d28a8721..3b5d398e 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -1,4 +1,14 @@ import { useCustomSource } from '@fishjam-cloud/react-native-client'; +import { useManagedPooledTrack } from '@fishjam-cloud/react-native-custom-video-source'; +import { + type CameraShaderBindings, + createCameraBindGroup, + getOutputSurfaceFormat, + getWebGpuRuntime, + useCameraWebGpuDeviceWithOverride, + type WebGpuFrameRenderContext, + type WebGpuFrameRenderFunction, +} from '@fishjam-cloud/react-native-custom-video-source/webgpu'; import { type MediaStream, pushFrame } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useMemo } from 'react'; import { @@ -11,13 +21,7 @@ import { import { type GPUSharedTextureMemory, GPUTextureUsage } from 'react-native-webgpu'; import { createFrameTimestampState, nextFrameTimestampNanoseconds } from '../frameTimestamp'; -import { useManagedPooledTrack } from '../internal/useManagedPooledTrack'; import { rotationDegreesFromOrientation } from '../orientation'; -import { type CameraShaderBindings, createCameraBindGroup } from './cameraShaderBindings'; -import type { WebGpuFrameRenderContext, WebGpuFrameRenderFunction } from './frameRenderContext'; -import { getOutputSurfaceFormat } from './requiredFeatures'; -import { useCameraWebGpuDeviceWithOverride } from './useCameraWebGpuDevice'; -import { getWebGpuRuntime } from './webGpuRuntime'; const DEFAULT_POOL_SIZE = 3; const DEFAULT_FRAME_INTERVAL_NANOSECONDS = 33_333_333; // 30 fps fallback cadence diff --git a/yarn.lock b/yarn.lock index 8f09c4e6..d1b16511 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5345,6 +5345,36 @@ __metadata: languageName: unknown linkType: soft +"@fishjam-cloud/react-native-custom-video-source@workspace:*, @fishjam-cloud/react-native-custom-video-source@workspace:packages/react-native-custom-video-source": + version: 0.0.0-use.local + resolution: "@fishjam-cloud/react-native-custom-video-source@workspace:packages/react-native-custom-video-source" + dependencies: + "@babel/core": "npm:^7.29.0" + "@babel/preset-typescript": "npm:^7.27.0" + "@fishjam-cloud/react-native-webrtc": "workspace:*" + "@types/react": "npm:19.2.14" + "@webgpu/types": "npm:0.1.65" + eslint-config-expo: "npm:~9.2.0" + eslint-import-resolver-typescript: "npm:^3.10.1" + eslint-plugin-prettier: "npm:^5.5.1" + react: "npm:19.2.3" + react-native: "npm:0.85.3" + react-native-builder-bob: "npm:^0.18.2" + react-native-webgpu: "npm:0.5.15" + typegpu: "npm:^0.11.8" + typescript: "npm:^5.8.3" + unplugin-typegpu: "npm:^0.11.5" + peerDependencies: + "@fishjam-cloud/react-native-webrtc": "workspace:*" + react: "*" + react-native: "*" + react-native-webgpu: ">=0.5.15" + peerDependenciesMeta: + react-native-webgpu: + optional: true + languageName: unknown + linkType: soft + "@fishjam-cloud/react-native-vision-camera-source@workspace:packages/react-native-vision-camera-source": version: 0.0.0-use.local resolution: "@fishjam-cloud/react-native-vision-camera-source@workspace:packages/react-native-vision-camera-source" @@ -5352,6 +5382,7 @@ __metadata: "@babel/core": "npm:^7.29.0" "@babel/preset-typescript": "npm:^7.27.0" "@fishjam-cloud/react-native-client": "workspace:*" + "@fishjam-cloud/react-native-custom-video-source": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*" "@types/react": "npm:19.2.14" "@webgpu/types": "npm:0.1.65" @@ -5366,9 +5397,7 @@ __metadata: react-native-vision-camera-worklets: "npm:5.0.10" react-native-webgpu: "npm:0.5.15" react-native-worklets: "npm:>=0.9.0" - typegpu: "npm:^0.11.8" typescript: "npm:^5.8.3" - unplugin-typegpu: "npm:^0.11.5" peerDependencies: "@fishjam-cloud/react-native-client": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*" From 7119205a83a6b9933ce0320d37048dafe68af2fc Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 12:25:36 +0200 Subject: [PATCH 16/19] Apply PR #560 review fixes - release: publish + version-bump react-native-custom-video-source, gate vision-camera-source publish on it - vision-camera-source: react-native-worklets as peer dep (native module) - webgpu: guard shared-device slot against late device-lost callbacks - docs: fix stale package name in RNWebGPU warning, README outputView example, encodeCameraPassthrough single-crop constraint, timestamp fallback quirk, drop untrue "asserted in tests" claim - extract shared usePublishedStream + stopStreamTracks helpers - log frame-processing failures with the cause object Co-Authored-By: Claude Fable 5 --- .github/workflows/release.yaml | 26 ++++++++++ .../hooks/internal/useCustomSourceManager.ts | 4 ++ .../src/internal/stopStreamTracks.ts | 10 ++++ .../src/internal/useManagedForwardTrack.ts | 13 ++--- .../src/internal/useManagedPooledTrack.ts | 7 +-- .../src/webgpu/cameraPassthroughPipeline.ts | 5 ++ .../src/webgpu/cropUtilities.ts | 6 +-- .../src/webgpu/useCameraWebGpuDevice.ts | 33 +++++++----- .../src/webgpu/webGpuRuntime.ts | 2 +- .../README.md | 6 ++- .../package.json | 7 +-- .../src/frameTimestamp.ts | 5 ++ .../src/internal/usePublishedStream.ts | 15 ++++++ .../src/useVisionCameraSource.ts | 17 ++----- .../src/webgpu/useVisionCameraWebGpuSource.ts | 18 ++----- release-automation/bump-version.sh | 3 ++ yarn.lock | 51 ++++++++++--------- 17 files changed, 140 insertions(+), 88 deletions(-) create mode 100644 packages/react-native-custom-video-source/src/internal/stopStreamTracks.ts create mode 100644 packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 30a935f5..43b869a1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -92,7 +92,33 @@ jobs: - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }} working-directory: packages/mobile-client + publish-react-native-custom-video-source: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + - run: corepack enable + - run: yarn + - run: yarn workspace @fishjam-cloud/react-native-custom-video-source run check:webrtc-published + - run: yarn build + - run: yarn pack --out package.tgz + working-directory: packages/react-native-custom-video-source + - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }} + working-directory: packages/react-native-custom-video-source + publish-react-native-vision-camera-source: + # vision-camera-source depends on custom-video-source at the same version, so it must not + # reach npm unless that dependency's publish succeeded. + needs: publish-react-native-custom-video-source runs-on: ubuntu-latest permissions: contents: read diff --git a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts index 2d0b10e0..f3714bef 100644 --- a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts +++ b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts @@ -94,6 +94,10 @@ export function useCustomSourceManager({ const setStream = useCallback( async (sourceId: string, stream: MediaStream | null) => { + // Note: the ref only advances on re-render, so two calls for the same source in the same + // tick both read the same snapshot and both try to remove the same track IDs; the loser's + // removeTracks rejects ("Cannot find "). Pre-existing behavior — the old + // closure-captured `sources` had the same window. const oldSource = sourcesRef.current[sourceId]; if (stream === oldSource?.stream) return; diff --git a/packages/react-native-custom-video-source/src/internal/stopStreamTracks.ts b/packages/react-native-custom-video-source/src/internal/stopStreamTracks.ts new file mode 100644 index 00000000..a3108077 --- /dev/null +++ b/packages/react-native-custom-video-source/src/internal/stopStreamTracks.ts @@ -0,0 +1,10 @@ +import type { MediaStream } from '@fishjam-cloud/react-native-webrtc'; + +/** Stops every track of `stream`, downgrading failures to a warning tagged with `ownerLabel`. */ +export function stopStreamTracks(stream: MediaStream, ownerLabel: string): void { + try { + stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); + } catch (cause) { + console.warn(`${ownerLabel}: stopping tracks failed`, cause); + } +} diff --git a/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts index b338eede..421058ed 100644 --- a/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts +++ b/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts @@ -6,6 +6,7 @@ import { } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useState } from 'react'; +import { stopStreamTracks } from './stopStreamTracks'; import { toError } from './toError'; interface ManagedForwardTrack { @@ -16,14 +17,6 @@ interface ManagedForwardTrack { const INITIAL_STATE: ManagedForwardTrack = { track: null, stream: null, error: null }; -function stopStreamTracks(stream: MediaStream): void { - try { - stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); - } catch (cause) { - console.warn('useManagedForwardTrack: stopping tracks failed', cause); - } -} - /** * Owns the async lifecycle of a forwarding custom video track: creates it while `enabled`, * exposes the handle + stream once ready, and stops the tracks on disable/unmount (also when @@ -46,7 +39,7 @@ export function useManagedForwardTrack(enabled: boolean): ManagedForwardTrack { .then((result) => { if (disposed) { // Torn down while creating — throw the just-built track away. - stopStreamTracks(result.stream); + stopStreamTracks(result.stream, 'useManagedForwardTrack'); return; } created = result; @@ -61,7 +54,7 @@ export function useManagedForwardTrack(enabled: boolean): ManagedForwardTrack { return () => { disposed = true; if (created != null) { - stopStreamTracks(created.stream); + stopStreamTracks(created.stream, 'useManagedForwardTrack'); } setManagedTrack(INITIAL_STATE); }; diff --git a/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts index 217ae648..9216175f 100644 --- a/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts +++ b/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts @@ -8,6 +8,7 @@ import { } from '@fishjam-cloud/react-native-webrtc'; import { useEffect, useState } from 'react'; +import { stopStreamTracks } from './stopStreamTracks'; import { toError } from './toError'; /** One pooled output surface as plain values the frame worklet can capture and import itself. */ @@ -48,11 +49,7 @@ function disposePool(pool: CustomVideoBufferPool): void { /** Tears an allocation down in the required order: stop the track's frames, then free its pool. */ function disposeAllocation({ pool, stream }: PooledTrackAllocation): void { - try { - stream.getTracks().forEach((mediaTrack) => mediaTrack.stop()); - } catch (cause) { - console.warn('useManagedPooledTrack: stopping tracks failed', cause); - } + stopStreamTracks(stream, 'useManagedPooledTrack'); disposePool(pool); } diff --git a/packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts b/packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts index 09dd5a8b..42fc8542 100644 --- a/packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts +++ b/packages/react-native-custom-video-source/src/webgpu/cameraPassthroughPipeline.ts @@ -153,6 +153,11 @@ export function createCameraPassthroughPipeline( * Worklet-safe; call it inside your render callback, and encode any overlay passes after it on * the same command encoder (with `loadOp: 'load'` so they draw on top). * + * At most one call per pipeline instance per frame: the crop lives in a single uniform buffer + * written via `queue.writeBuffer`, which lands before the submitted command buffer executes — a + * second call in the same frame makes both draws use the second crop. Encode to multiple targets + * with different crops by building one pipeline per target. + * * @group WebGPU */ export function encodeCameraPassthrough( diff --git a/packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts b/packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts index 4b6c22eb..d3383645 100644 --- a/packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts +++ b/packages/react-native-custom-video-source/src/webgpu/cropUtilities.ts @@ -42,9 +42,9 @@ export interface FrameCrop { } // Derived from the schema so the buffer size can never drift from the WGSL struct. The manual -// offsets below match the schema's std140 layout (asserted in tests); we keep a hand-written -// packer rather than a schema-driven writer because this runs per-frame inside a worklet and must -// stay off the tgpu root/proxy. +// offsets below must match the schema's std140 layout by hand (see the layout table on +// FrameCropParams); we keep a hand-written packer rather than a schema-driven writer because this +// runs per-frame inside a worklet and must stay off the tgpu root/proxy. const FRAME_CROP_BYTES = d.sizeOf(FrameCropParams); /** diff --git a/packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts b/packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts index 7a4cf875..be8ed54b 100644 --- a/packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts +++ b/packages/react-native-custom-video-source/src/webgpu/useCameraWebGpuDevice.ts @@ -15,25 +15,32 @@ async function acquireSharedCameraWebGpuDevice(): Promise { if (adapter == null) { throw new Error('WebGPU is unavailable: navigator.gpu.requestAdapter() returned no adapter.'); } - const device = await adapter.requestDevice({ + return adapter.requestDevice({ requiredFeatures: getRequiredWebGpuCameraFeatures(), }); - device.lost - .then(() => { - sharedDevicePromise = null; - }) - .catch(() => { - sharedDevicePromise = null; - }); - return device; } function getSharedCameraWebGpuDevice(): Promise { if (sharedDevicePromise == null) { - sharedDevicePromise = acquireSharedCameraWebGpuDevice().catch((cause: unknown) => { - sharedDevicePromise = null; - throw cause; - }); + // Only this promise may clear the slot: by the time a loss (or failure) callback fires, a + // replacement device may already occupy it, and unconditionally nulling would discard that + // replacement. + const clearIfCurrent = () => { + if (sharedDevicePromise === devicePromise) { + sharedDevicePromise = null; + } + }; + const devicePromise: Promise = acquireSharedCameraWebGpuDevice().then( + (device) => { + device.lost.then(clearIfCurrent, clearIfCurrent); + return device; + }, + (cause: unknown) => { + clearIfCurrent(); + throw cause; + }, + ); + sharedDevicePromise = devicePromise; } return sharedDevicePromise; } diff --git a/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts b/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts index a6eb3009..9cbaf000 100644 --- a/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts +++ b/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts @@ -13,7 +13,7 @@ export function getWebGpuRuntime(): WebGpuRuntime { if (typeof RNWebGPU === 'undefined') { const installedOk = WebGPUModule.install(); console.warn( - `react-native-vision-camera-source: the RNWebGPU global was missing on this runtime; ` + + `react-native-custom-video-source: the RNWebGPU global was missing on this runtime; ` + `re-ran install() -> ${String(installedOk)}`, ); } diff --git a/packages/react-native-vision-camera-source/README.md b/packages/react-native-vision-camera-source/README.md index 80a4fcf9..73ab15f7 100644 --- a/packages/react-native-vision-camera-source/README.md +++ b/packages/react-native-vision-camera-source/README.md @@ -118,9 +118,11 @@ const onFrame = useCallback( (frame: Frame, render: WebGpuFrameRenderFunction) => { 'worklet'; if (effect == null) return; // drop until the pipeline is ready - render(({ commandEncoder, outputTexture, cameraBindGroup }) => { + render(({ commandEncoder, outputView, cameraBindGroup }) => { + // Use the provided outputView — a per-frame outputTexture.createView() would leak native + // wrappers on the frame runtime (GPUTextureView has no release API). const pass = commandEncoder.beginRenderPass({ - colorAttachments: [{ view: outputTexture.createView(), loadOp: 'clear', storeOp: 'store' }], + colorAttachments: [{ view: outputView, loadOp: 'clear', storeOp: 'store' }], }); pass.setPipeline(effect.pipeline); pass.setBindGroup(0, cameraBindGroup!); diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json index e16011d0..06480358 100644 --- a/packages/react-native-vision-camera-source/package.json +++ b/packages/react-native-vision-camera-source/package.json @@ -82,7 +82,8 @@ "react-native": "*", "react-native-vision-camera": "^5.0.0", "react-native-vision-camera-worklets": "^5.0.0", - "react-native-webgpu": ">=0.5.15" + "react-native-webgpu": ">=0.5.15", + "react-native-worklets": ">=0.9.0" }, "peerDependenciesMeta": { "react-native-webgpu": { @@ -90,8 +91,7 @@ } }, "dependencies": { - "@fishjam-cloud/react-native-custom-video-source": "workspace:*", - "react-native-worklets": ">=0.9.0" + "@fishjam-cloud/react-native-custom-video-source": "workspace:*" }, "devDependencies": { "@babel/core": "^7.29.0", @@ -110,6 +110,7 @@ "react-native-vision-camera": "5.0.10", "react-native-vision-camera-worklets": "5.0.10", "react-native-webgpu": "0.5.15", + "react-native-worklets": "0.10.2", "typescript": "^5.8.3" }, "packageManager": "yarn@4.16.0" diff --git a/packages/react-native-vision-camera-source/src/frameTimestamp.ts b/packages/react-native-vision-camera-source/src/frameTimestamp.ts index db59cf26..0dad66e1 100644 --- a/packages/react-native-vision-camera-source/src/frameTimestamp.ts +++ b/packages/react-native-vision-camera-source/src/frameTimestamp.ts @@ -59,6 +59,11 @@ export function normalizeFrameTimestampNanoseconds(state: FrameTimestampState, r * Like {@link normalizeFrameTimestampNanoseconds}, but never fails: when the frame carries no * usable timestamp, it advances the timeline by `frameIntervalNanoseconds` instead. For sinks * that require a timestamp on every frame (`pushFrame`). + * + * Known quirk, deliberately accepted: if the first frames carry no timestamp (interval-paced) and + * real timestamps then appear, the baseline is taken from the first real one, so real values start + * at 0 — behind the interval-paced `lastNanoseconds` — and are rejected until the real timeline + * catches up. Output stays monotonic throughout; the cost is a short stretch of synthesized pacing. */ export function nextFrameTimestampNanoseconds( state: FrameTimestampState, diff --git a/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts new file mode 100644 index 00000000..84d37bb9 --- /dev/null +++ b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts @@ -0,0 +1,15 @@ +import { useCustomSource } from '@fishjam-cloud/react-native-client'; +import type { MediaStream } from '@fishjam-cloud/react-native-webrtc'; +import { useEffect } from 'react'; + +/** Publishes `stream` under `sourceId` while it exists, unpublishing on change/cleanup/unmount. */ +export function usePublishedStream(sourceId: string, stream: MediaStream | null): void { + const { setStream } = useCustomSource(sourceId); + useEffect(() => { + if (stream == null) return; + void setStream(stream); + return () => { + void setStream(null); + }; + }, [stream, setStream]); +} diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts index c5384557..bcbe0b3e 100644 --- a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -1,7 +1,6 @@ -import { useCustomSource } from '@fishjam-cloud/react-native-client'; import { useManagedForwardTrack } from '@fishjam-cloud/react-native-custom-video-source'; import { forwardFrame, type MediaStream } from '@fishjam-cloud/react-native-webrtc'; -import { useEffect, useMemo } from 'react'; +import { useMemo } from 'react'; import { type CameraFrameOutput, type Frame, @@ -11,6 +10,7 @@ import { } from 'react-native-vision-camera'; import { createFrameTimestampState, normalizeFrameTimestampNanoseconds } from './frameTimestamp'; +import { usePublishedStream } from './internal/usePublishedStream'; import { rotationDegreesFromOrientation } from './orientation'; /** @@ -84,16 +84,7 @@ export function useVisionCameraSource( const { enabled = true, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; const { track, stream, error } = useManagedForwardTrack(enabled); - - // Publish the track's stream under sourceId while it exists, unpublish on cleanup/unmount. - const { setStream } = useCustomSource(sourceId); - useEffect(() => { - if (stream == null) return; - void setStream(stream); - return () => { - void setStream(null); - }; - }, [stream, setStream]); + usePublishedStream(sourceId, stream); const timestampState = useMemo(() => createFrameTimestampState(), []); @@ -120,7 +111,7 @@ export function useVisionCameraSource( userOnFrame(frame); } } catch (cause) { - console.warn('useVisionCameraSource: processing a camera frame failed: ' + String(cause)); + console.warn('useVisionCameraSource: processing a camera frame failed', cause); } finally { frame.dispose(); } diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index 3b5d398e..6c92d494 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -1,4 +1,3 @@ -import { useCustomSource } from '@fishjam-cloud/react-native-client'; import { useManagedPooledTrack } from '@fishjam-cloud/react-native-custom-video-source'; import { type CameraShaderBindings, @@ -10,7 +9,7 @@ import { type WebGpuFrameRenderFunction, } from '@fishjam-cloud/react-native-custom-video-source/webgpu'; import { type MediaStream, pushFrame } from '@fishjam-cloud/react-native-webrtc'; -import { useEffect, useMemo } from 'react'; +import { useMemo } from 'react'; import { type CameraFrameOutput, type Frame, @@ -21,6 +20,7 @@ import { import { type GPUSharedTextureMemory, GPUTextureUsage } from 'react-native-webgpu'; import { createFrameTimestampState, nextFrameTimestampNanoseconds } from '../frameTimestamp'; +import { usePublishedStream } from '../internal/usePublishedStream'; import { rotationDegreesFromOrientation } from '../orientation'; const DEFAULT_POOL_SIZE = 3; @@ -144,16 +144,7 @@ export function useVisionCameraWebGpuSource( bufferDescriptors, error: trackError, } = useManagedPooledTrack(enabled, width, height, poolSize); - - // Publish the track's stream under sourceId while it exists, unpublish on cleanup/unmount. - const { setStream } = useCustomSource(sourceId); - useEffect(() => { - if (stream == null) return; - void setStream(stream); - return () => { - void setStream(null); - }; - }, [stream, setStream]); + usePublishedStream(sourceId, stream); const runtime = getWebGpuRuntime(); const outputSurfaceFormat = getOutputSurfaceFormat(); @@ -170,6 +161,7 @@ export function useVisionCameraWebGpuSource( // deterministic alternative (import + destroy every frame) costs an import per frame — switch // to it if leak measurements ever demand. const workletState = useMemo(() => { + // track/device are not read here — the `void`s mark them as intentional reset-only deps. void track; void device; return { @@ -297,7 +289,7 @@ export function useVisionCameraWebGpuSource( nativeBuffer.release(); } } catch (cause) { - console.warn('useVisionCameraWebGpuSource: processing a camera frame failed: ' + String(cause)); + console.warn('useVisionCameraWebGpuSource: processing a camera frame failed', cause); } finally { frame.dispose(); } diff --git a/release-automation/bump-version.sh b/release-automation/bump-version.sh index ebee20cb..66e51def 100755 --- a/release-automation/bump-version.sh +++ b/release-automation/bump-version.sh @@ -59,6 +59,9 @@ echo "Updated react-client to $VERSION" corepack yarn workspace @fishjam-cloud/react-native-client version "$VERSION" echo "Updated react-native-client to $VERSION" +corepack yarn workspace @fishjam-cloud/react-native-custom-video-source version "$VERSION" +echo "Updated react-native-custom-video-source to $VERSION" + corepack yarn workspace @fishjam-cloud/react-native-vision-camera-source version "$VERSION" echo "Updated react-native-vision-camera-source to $VERSION" diff --git a/yarn.lock b/yarn.lock index d1b16511..060692a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5396,7 +5396,7 @@ __metadata: react-native-vision-camera: "npm:5.0.10" react-native-vision-camera-worklets: "npm:5.0.10" react-native-webgpu: "npm:0.5.15" - react-native-worklets: "npm:>=0.9.0" + react-native-worklets: "npm:0.10.2" typescript: "npm:^5.8.3" peerDependencies: "@fishjam-cloud/react-native-client": "workspace:*" @@ -5406,6 +5406,7 @@ __metadata: react-native-vision-camera: ^5.0.0 react-native-vision-camera-worklets: ^5.0.0 react-native-webgpu: ">=0.5.15" + react-native-worklets: ">=0.9.0" peerDependenciesMeta: react-native-webgpu: optional: true @@ -18136,30 +18137,7 @@ __metadata: languageName: node linkType: hard -"react-native-worklets@npm:0.5.1": - version: 0.5.1 - resolution: "react-native-worklets@npm:0.5.1" - dependencies: - "@babel/plugin-transform-arrow-functions": "npm:^7.0.0-0" - "@babel/plugin-transform-class-properties": "npm:^7.0.0-0" - "@babel/plugin-transform-classes": "npm:^7.0.0-0" - "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.0.0-0" - "@babel/plugin-transform-optional-chaining": "npm:^7.0.0-0" - "@babel/plugin-transform-shorthand-properties": "npm:^7.0.0-0" - "@babel/plugin-transform-template-literals": "npm:^7.0.0-0" - "@babel/plugin-transform-unicode-regex": "npm:^7.0.0-0" - "@babel/preset-typescript": "npm:^7.16.7" - convert-source-map: "npm:^2.0.0" - semver: "npm:7.7.2" - peerDependencies: - "@babel/core": ^7.0.0-0 - react: "*" - react-native: "*" - checksum: 10c0/9eb9e6dea9abaf889400a6618355ef59af3075f5004a4bec9e4cba6dcfd13d8b63de0d4b29d75c00a3dcf5ad422e1bdb71636c75b1a2ad1c43d8b512f198bdab - languageName: node - linkType: hard - -"react-native-worklets@npm:>=0.9.0": +"react-native-worklets@npm:0.10.2": version: 0.10.2 resolution: "react-native-worklets@npm:0.10.2" dependencies: @@ -18184,6 +18162,29 @@ __metadata: languageName: node linkType: hard +"react-native-worklets@npm:0.5.1": + version: 0.5.1 + resolution: "react-native-worklets@npm:0.5.1" + dependencies: + "@babel/plugin-transform-arrow-functions": "npm:^7.0.0-0" + "@babel/plugin-transform-class-properties": "npm:^7.0.0-0" + "@babel/plugin-transform-classes": "npm:^7.0.0-0" + "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.0.0-0" + "@babel/plugin-transform-optional-chaining": "npm:^7.0.0-0" + "@babel/plugin-transform-shorthand-properties": "npm:^7.0.0-0" + "@babel/plugin-transform-template-literals": "npm:^7.0.0-0" + "@babel/plugin-transform-unicode-regex": "npm:^7.0.0-0" + "@babel/preset-typescript": "npm:^7.16.7" + convert-source-map: "npm:^2.0.0" + semver: "npm:7.7.2" + peerDependencies: + "@babel/core": ^7.0.0-0 + react: "*" + react-native: "*" + checksum: 10c0/9eb9e6dea9abaf889400a6618355ef59af3075f5004a4bec9e4cba6dcfd13d8b63de0d4b29d75c00a3dcf5ad422e1bdb71636c75b1a2ad1c43d8b512f198bdab + languageName: node + linkType: hard + "react-native@npm:0.81.5": version: 0.81.5 resolution: "react-native@npm:0.81.5" From 4d6a10817acff669c4a41f33e3952e1e6b27bdc8 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 12:42:06 +0200 Subject: [PATCH 17/19] Fix docs generation failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export ManagedForwardTrack/ManagedPooledTrack — they are the return types of public hooks, so typedoc (treatWarningsAsErrors) failed on them being absent from the docs. Exclude external symbols in vision-camera-source docs: the options interfaces inherit VisionCamera members whose upstream doc comments carry @link tags unresolvable in our docs. Also turn the one cross-entry-point @link in webgpu.ts into a plain reference. Co-Authored-By: Claude Fable 5 --- packages/react-native-custom-video-source/src/index.ts | 8 ++++++-- .../src/internal/useManagedForwardTrack.ts | 6 +++++- .../src/internal/useManagedPooledTrack.ts | 6 +++++- packages/react-native-custom-video-source/src/webgpu.ts | 2 +- packages/react-native-vision-camera-source/typedoc.json | 3 ++- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/react-native-custom-video-source/src/index.ts b/packages/react-native-custom-video-source/src/index.ts index 50aaa1c1..ddd57a7e 100644 --- a/packages/react-native-custom-video-source/src/index.ts +++ b/packages/react-native-custom-video-source/src/index.ts @@ -17,5 +17,9 @@ * @packageDocumentation */ -export { useManagedForwardTrack } from './internal/useManagedForwardTrack'; -export { useManagedPooledTrack, type WorkletBufferDescriptor } from './internal/useManagedPooledTrack'; +export { type ManagedForwardTrack, useManagedForwardTrack } from './internal/useManagedForwardTrack'; +export { + type ManagedPooledTrack, + useManagedPooledTrack, + type WorkletBufferDescriptor, +} from './internal/useManagedPooledTrack'; diff --git a/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts index 421058ed..5918dfbd 100644 --- a/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts +++ b/packages/react-native-custom-video-source/src/internal/useManagedForwardTrack.ts @@ -9,7 +9,11 @@ import { useEffect, useState } from 'react'; import { stopStreamTracks } from './stopStreamTracks'; import { toError } from './toError'; -interface ManagedForwardTrack { +/** + * State of a forwarding custom video track managed by {@link useManagedForwardTrack}. + * While the track is being created (or after an error) `track` and `stream` are `null`. + */ +export interface ManagedForwardTrack { track: ForwardTrack | null; stream: MediaStream | null; error: Error | null; diff --git a/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts b/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts index 9216175f..ab12f916 100644 --- a/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts +++ b/packages/react-native-custom-video-source/src/internal/useManagedPooledTrack.ts @@ -19,7 +19,11 @@ export interface WorkletBufferDescriptor { height: number; } -interface ManagedPooledTrack { +/** + * State of a pooled custom video track managed by {@link useManagedPooledTrack}. + * While the pool and track are being created (or after an error) all fields are `null`. + */ +export interface ManagedPooledTrack { track: PooledTrack | null; stream: MediaStream | null; /** Plain per-surface descriptors (the pool object itself is not worklet-serializable). */ diff --git a/packages/react-native-custom-video-source/src/webgpu.ts b/packages/react-native-custom-video-source/src/webgpu.ts index 32c0aa0e..1f5a8525 100644 --- a/packages/react-native-custom-video-source/src/webgpu.ts +++ b/packages/react-native-custom-video-source/src/webgpu.ts @@ -1,7 +1,7 @@ /// /** * WebGPU camera toolkit — sample the live camera and render your own content into the video you - * publish with {@link useManagedPooledTrack}. Requires `react-native-webgpu` (an optional peer of + * publish with `useManagedPooledTrack` (from the package's main entry point). Requires `react-native-webgpu` (an optional peer of * this package; only this entry point loads it). * * - {@link createCameraShaderBindings} — sample the live camera from your own shaders via diff --git a/packages/react-native-vision-camera-source/typedoc.json b/packages/react-native-vision-camera-source/typedoc.json index b14f5ae9..1405eabb 100644 --- a/packages/react-native-vision-camera-source/typedoc.json +++ b/packages/react-native-vision-camera-source/typedoc.json @@ -2,5 +2,6 @@ "$schema": "https://typedoc.org/schema.json", "includeVersion": true, "entryPoints": ["src/index.ts", "src/webgpu.ts"], - "readme": "none" + "readme": "none", + "excludeExternals": true } From cdb36e46a7ede8c5fe47cfa2d56917bfbf4b656e Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 12:56:26 +0200 Subject: [PATCH 18/19] Fix timestamp unit misclassification, unhandled setStream rejections, and silent RNWebGPU failure - Lower the frame-timestamp magnitude threshold from 1e12 to 1e9: both platform clocks count from boot, so seconds can never reach 1e9 while nanoseconds pass it one second after boot. The old threshold misclassified Android frames captured within ~16.7 minutes of boot. - Catch setStream rejections in usePublishedStream (the same-tick removeTracks race rejects) and downgrade them to warnings. - Make getWebGpuRuntime throw a descriptive error when install() cannot bind the RNWebGPU global, and surface that through the WebGPU source hook's `error` result instead of crashing render or failing per frame. - Add vitest + a frameTimestamp regression suite (fails on the old threshold, passes on the fix). Co-Authored-By: Claude Fable 5 --- .../src/webgpu/webGpuRuntime.ts | 16 ++- .../package.json | 4 +- .../src/frameTimestamp.ts | 15 ++- .../src/internal/usePublishedStream.ts | 10 +- .../src/webgpu/useVisionCameraWebGpuSource.ts | 16 ++- .../tests/frameTimestamp.test.ts | 105 ++++++++++++++++++ yarn.lock | 1 + 7 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts diff --git a/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts b/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts index 9cbaf000..145fb1c4 100644 --- a/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts +++ b/packages/react-native-custom-video-source/src/webgpu/webGpuRuntime.ts @@ -8,14 +8,22 @@ export type WebGpuRuntime = typeof RNWebGPU; * missing on the calling runtime. The library binds the global once at module load, but that * binding can be absent after a crash-recovery runtime reload or an install-ordering flake; * `WebGPUModule.install()` binds it to the calling runtime and is safe to repeat. + * + * @throws When the runtime cannot be installed (react-native-webgpu missing or its native module + * not linked) — returning the still-undefined global would only defer the failure to an opaque + * per-frame TypeError. */ export function getWebGpuRuntime(): WebGpuRuntime { if (typeof RNWebGPU === 'undefined') { const installedOk = WebGPUModule.install(); - console.warn( - `react-native-custom-video-source: the RNWebGPU global was missing on this runtime; ` + - `re-ran install() -> ${String(installedOk)}`, - ); + if (!installedOk || typeof RNWebGPU === 'undefined') { + throw new Error( + 'react-native-custom-video-source: the RNWebGPU global is missing and WebGPUModule.install() ' + + 'could not bind it. Make sure react-native-webgpu is installed and its native module is linked ' + + '(rebuild the app after adding it).', + ); + } + console.warn('react-native-custom-video-source: the RNWebGPU global was missing on this runtime; re-ran install()'); } return RNWebGPU; } diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json index 06480358..46f758ff 100644 --- a/packages/react-native-vision-camera-source/package.json +++ b/packages/react-native-vision-camera-source/package.json @@ -41,6 +41,7 @@ ], "scripts": { "tsc": "tsc --noEmit", + "test": "vitest run", "build:webrtc-workspace": "yarn workspace @fishjam-cloud/react-native-webrtc run prepare", "build": "yarn build:webrtc-workspace && bob build", "lint": "eslint . --ext .ts,.tsx --fix", @@ -111,7 +112,8 @@ "react-native-vision-camera-worklets": "5.0.10", "react-native-webgpu": "0.5.15", "react-native-worklets": "0.10.2", - "typescript": "^5.8.3" + "typescript": "^5.8.3", + "vitest": "^3.1.1" }, "packageManager": "yarn@4.16.0" } diff --git a/packages/react-native-vision-camera-source/src/frameTimestamp.ts b/packages/react-native-vision-camera-source/src/frameTimestamp.ts index 0dad66e1..6d6acb5e 100644 --- a/packages/react-native-vision-camera-source/src/frameTimestamp.ts +++ b/packages/react-native-vision-camera-source/src/frameTimestamp.ts @@ -1,12 +1,15 @@ import { createSynchronizable, type Synchronizable } from 'react-native-worklets'; // VisionCamera 5.x reports Frame.timestamp in platform-inconsistent units: seconds on iOS -// (CMTime.seconds) and nanoseconds on Android (the raw CameraX sensor timestamp). Rather than -// branching on the platform — which would silently break if VisionCamera ever aligns the units — -// we discriminate by magnitude: seconds-since-boot values sit around 1e5–1e6 while -// nanoseconds-since-boot values sit around 1e14, so any value at or above this threshold is -// already in nanoseconds. -const NANOSECONDS_MAGNITUDE_THRESHOLD = 1e12; +// (CMTime.seconds) and nanoseconds on Android (the raw CameraX ImageInfo.getTimestamp() — its +// HybridPhoto divides by 1e9 but HybridFrame does not). Rather than branching on the platform — +// which would silently break if VisionCamera ever aligns the units — we discriminate by +// magnitude. Both clocks count from boot, so 1e9 separates them cleanly: a value in seconds +// cannot reach it (1e9 seconds ≈ 31.7 years of uptime), while a value in nanoseconds exceeds it +// one second after boot — long before any camera can deliver a frame. Any value at or above this +// threshold is therefore already in nanoseconds. (An earlier 1e12 threshold misclassified Android +// frames captured within ~16.7 minutes of boot as seconds.) +const NANOSECONDS_MAGNITUDE_THRESHOLD = 1e9; const NANOSECONDS_PER_SECOND = 1e9; diff --git a/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts index 84d37bb9..0d6a0ccd 100644 --- a/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts +++ b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts @@ -7,9 +7,15 @@ export function usePublishedStream(sourceId: string, stream: MediaStream | null) const { setStream } = useCustomSource(sourceId); useEffect(() => { if (stream == null) return; - void setStream(stream); + // setStream can reject (e.g. two publishes racing over the same source remove the same stale + // track IDs); downgrade to a warning — an unhandled rejection here would take down dev builds. + setStream(stream).catch((cause: unknown) => { + console.warn('usePublishedStream: publishing the stream failed', cause); + }); return () => { - void setStream(null); + setStream(null).catch((cause: unknown) => { + console.warn('usePublishedStream: unpublishing the stream failed', cause); + }); }; }, [stream, setStream]); } diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index 6c92d494..d679eb5e 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -88,7 +88,7 @@ export interface UseVisionCameraWebGpuSourceResult { stream: MediaStream | null; /** The GPUDevice in use — build your pipelines against it. `null` until acquired. */ device: GPUDevice | null; - /** Track creation, publishing, or device acquisition failure, if any. */ + /** WebGPU runtime, device acquisition, track creation, or publishing failure, if any. */ error: Error | null; } @@ -146,7 +146,15 @@ export function useVisionCameraWebGpuSource( } = useManagedPooledTrack(enabled, width, height, poolSize); usePublishedStream(sourceId, stream); - const runtime = getWebGpuRuntime(); + // getWebGpuRuntime throws when react-native-webgpu is missing/unlinked; surface that through + // the hook's `error` (like device/track failures) instead of crashing the component render. + const { runtime, runtimeError } = useMemo(() => { + try { + return { runtime: getWebGpuRuntime(), runtimeError: null }; + } catch (cause) { + return { runtime: null, runtimeError: cause instanceof Error ? cause : new Error(String(cause)) }; + } + }, []); const outputSurfaceFormat = getOutputSurfaceFormat(); // Captured as a plain number: the worklet must not close over the GPUTextureUsage namespace. const renderAttachmentUsage = GPUTextureUsage.RENDER_ATTACHMENT; @@ -177,7 +185,7 @@ export function useVisionCameraWebGpuSource( return (frame: Frame) => { 'worklet'; try { - if (track == null || bufferDescriptors == null || device == null) { + if (track == null || bufferDescriptors == null || device == null || runtime == null) { return; } const nativeBuffer = frame.getNativeBuffer(); @@ -318,5 +326,5 @@ export function useVisionCameraWebGpuSource( onFrameDropped, }); - return { frameOutput, stream, device, error: deviceError ?? trackError }; + return { frameOutput, stream, device, error: runtimeError ?? deviceError ?? trackError }; } diff --git a/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts b/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts new file mode 100644 index 00000000..0a0d489a --- /dev/null +++ b/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts @@ -0,0 +1,105 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + createFrameTimestampState, + type FrameTimestampState, + nextFrameTimestampNanoseconds, + normalizeFrameTimestampNanoseconds, +} from '../src/frameTimestamp'; + +// In-memory stand-in for react-native-worklets' Synchronizable (the real implementation needs a +// native runtime; vitest hoists this mock above the imports). Faithful to the surface +// frameTimestamp uses: getBlocking + setBlocking. +vi.mock('react-native-worklets', () => ({ + createSynchronizable: (initialValue: TValue) => { + let value = initialValue; + return { + getBlocking: () => value, + setBlocking: (next: TValue | ((previous: TValue) => TValue)) => { + value = typeof next === 'function' ? (next as (previous: TValue) => TValue)(value) : next; + }, + }; + }, +})); + +const NANOSECONDS_PER_SECOND = 1e9; + +let state: FrameTimestampState; +beforeEach(() => { + state = createFrameTimestampState(); +}); + +describe('normalizeFrameTimestampNanoseconds', () => { + it('rejects unusable timestamps (invalid frames report 0)', () => { + expect(normalizeFrameTimestampNanoseconds(state, 0)).toBeNull(); + expect(normalizeFrameTimestampNanoseconds(state, -5)).toBeNull(); + expect(normalizeFrameTimestampNanoseconds(state, Number.NaN)).toBeNull(); + }); + + it('treats iOS CMTime seconds as seconds and preserves inter-frame gaps in nanoseconds', () => { + // Typical iOS values: seconds since boot, ~1e5–1e6. + const firstSeconds = 123456.5; + const secondSeconds = 123456.5334; + expect(normalizeFrameTimestampNanoseconds(state, firstSeconds)).toBe(0); + expect(normalizeFrameTimestampNanoseconds(state, secondSeconds)).toBe( + secondSeconds * NANOSECONDS_PER_SECOND - firstSeconds * NANOSECONDS_PER_SECOND, + ); + }); + + it('treats Android CameraX nanoseconds as nanoseconds after long uptime', () => { + // ~28 hours of uptime, 33 ms apart. + const firstNanoseconds = 1e14; + expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds)).toBe(0); + expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds + 33_000_000)).toBe(33_000_000); + }); + + it('treats Android nanoseconds as nanoseconds on a recently booted device (regression)', () => { + // 2 minutes of uptime: 1.2e11 ns. The old 1e12 threshold misclassified this as seconds and + // scaled a 33 ms gap into ~380 days of timeline. + const firstNanoseconds = 1.2e11; + expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds)).toBe(0); + expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds + 33_000_000)).toBe(33_000_000); + }); + + it('stays continuous when an Android stream crosses a magnitude boundary mid-stream', () => { + // Frames straddling 1e12 ns (~16.7 min of uptime) — the old threshold flipped units here. + const beforeBoundary = 1e12 - 16_000_000; + const afterBoundary = 1e12 + 17_000_000; + expect(normalizeFrameTimestampNanoseconds(state, beforeBoundary)).toBe(0); + expect(normalizeFrameTimestampNanoseconds(state, afterBoundary)).toBe(33_000_000); + }); + + it('rejects timestamps that do not advance the timeline', () => { + expect(normalizeFrameTimestampNanoseconds(state, 1e14)).toBe(0); + expect(normalizeFrameTimestampNanoseconds(state, 1e14)).toBeNull(); + expect(normalizeFrameTimestampNanoseconds(state, 1e14 - 1_000_000)).toBeNull(); + // A later frame that does advance is accepted again. + expect(normalizeFrameTimestampNanoseconds(state, 1e14 + 1_000_000)).toBe(1_000_000); + }); +}); + +describe('nextFrameTimestampNanoseconds', () => { + const frameIntervalNanoseconds = 33_333_333; + + it('passes real timestamps through', () => { + expect(nextFrameTimestampNanoseconds(state, 1e14, frameIntervalNanoseconds)).toBe(0); + expect(nextFrameTimestampNanoseconds(state, 1e14 + 40_000_000, frameIntervalNanoseconds)).toBe(40_000_000); + }); + + it('paces by the frame interval when frames carry no timestamp', () => { + expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(0); + expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(frameIntervalNanoseconds); + expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(2 * frameIntervalNanoseconds); + }); + + it('stays monotonic when real timestamps appear after interval-paced frames (documented quirk)', () => { + const paced = nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds); + const pacedAgain = nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds); + // First real timestamp becomes the baseline (offset 0), which is behind the paced timeline — + // so it falls back to interval pacing rather than jumping backwards. + const afterRealTimestamp = nextFrameTimestampNanoseconds(state, 1e14, frameIntervalNanoseconds); + expect(paced).toBe(0); + expect(pacedAgain).toBeGreaterThan(paced); + expect(afterRealTimestamp).toBeGreaterThan(pacedAgain); + }); +}); diff --git a/yarn.lock b/yarn.lock index 060692a2..9b3f066a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5398,6 +5398,7 @@ __metadata: react-native-webgpu: "npm:0.5.15" react-native-worklets: "npm:0.10.2" typescript: "npm:^5.8.3" + vitest: "npm:^3.1.1" peerDependencies: "@fishjam-cloud/react-native-client": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*" From 7b1ea86dbe24ad054830bb7f6dd6c7c9f8d015a2 Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Fri, 10 Jul 2026 13:50:07 +0200 Subject: [PATCH 19/19] Serialize setStream and keep forwarded frames on one timestamp domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replacing a published stream issues an unpublish and a publish in the same tick; run concurrently they raced to remove the same track IDs and the new stream was nondeterministically never published. setStream calls now run through a promise chain, with a synchronously-updated sources snapshot so queued calls see each other's effects. The connected effect also no longer leaves addTrack rejections unhandled. useVisionCameraSource now supplies timestampNs on every frame (interval-paced fallback via nextFrameTimestampNanoseconds, like the WebGPU tier), clamped to at least 1 ns: native reads an omitted timestamp — and the value 0, the timeline's first frame — as "stamp with my own clock", which mixed two timestamp domains on one track. Co-Authored-By: Claude Fable 5 --- .../hooks/internal/useCustomSourceManager.ts | 74 +++++++++--- .../package.json | 4 +- .../src/frameTimestamp.ts | 3 + .../src/internal/usePublishedStream.ts | 5 +- .../src/useVisionCameraSource.ts | 33 ++++-- .../src/webgpu/useVisionCameraWebGpuSource.ts | 13 ++- .../tests/frameTimestamp.test.ts | 105 ------------------ yarn.lock | 1 - 8 files changed, 101 insertions(+), 137 deletions(-) delete mode 100644 packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts diff --git a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts index f3714bef..c2ef7d43 100644 --- a/packages/react-client/src/hooks/internal/useCustomSourceManager.ts +++ b/packages/react-client/src/hooks/internal/useCustomSourceManager.ts @@ -27,6 +27,12 @@ export function useCustomSourceManager({ const sourcesRef = useRef(sources); sourcesRef.current = sources; + // Replacing a source's stream issues two setStream calls in the same tick (unpublish the old, + // publish the new). Run concurrently they would read the same snapshot and race to remove the + // same track IDs — and when the publish call loses that race, the new stream is never + // published. This chain serializes the calls instead. + const setStreamQueue = useRef(Promise.resolve()); + const pendingSources = useMemo( () => Object.entries(sources).filter(([_, source]) => source.trackIds === undefined), [sources], @@ -92,51 +98,87 @@ export function useCustomSourceManager({ const getSource = useCallback((sourceId: string) => sources[sourceId], [sources]); - const setStream = useCallback( + // Updates both the state (what consumers render) and the ref (what queued setStream calls + // read synchronously, before React re-renders). + const updateSources = useCallback( + (update: (old: Record) => Record) => { + sourcesRef.current = update(sourcesRef.current); + setSources(update); + }, + [], + ); + + const applySetStream = useCallback( async (sourceId: string, stream: MediaStream | null) => { - // Note: the ref only advances on re-render, so two calls for the same source in the same - // tick both read the same snapshot and both try to remove the same track IDs; the loser's - // removeTracks rejects ("Cannot find "). Pre-existing behavior — the old - // closure-captured `sources` had the same window. const oldSource = sourcesRef.current[sourceId]; if (stream === oldSource?.stream) return; if (oldSource?.trackIds) await removeTracks(oldSource.trackIds); if (stream !== null) { - setSources((old) => ({ ...old, [sourceId]: { stream } })); - return; + updateSources((old) => ({ ...old, [sourceId]: { stream } })); + } else if (oldSource) { + updateSources((old) => Object.fromEntries(Object.entries(old).filter(([id]) => id !== sourceId))); } - if (!oldSource) return; + }, + [removeTracks, updateSources], + ); - setSources((old) => Object.fromEntries(Object.entries(old).filter(([id, _]) => id !== sourceId))); + const setStream = useCallback( + (sourceId: string, stream: MediaStream | null) => { + const run = setStreamQueue.current.then(() => applySetStream(sourceId, stream)); + // Chain a never-rejecting link so one failed call cannot poison the queue; the caller + // still observes failures through the returned promise. + setStreamQueue.current = run.catch(() => undefined); + return run; }, - [removeTracks], + [applySetStream], ); useEffect(() => { const onConnected = async () => { if (pendingSources.length === 0) return; - const patch = Object.fromEntries( - await Promise.all(pendingSources.map(async ([id, source]) => [id, await startStreaming(source)] as const)), + const results = await Promise.all( + pendingSources.map(async ([id, source]) => [id, await startStreaming(source)] as const), ); - setSources((old) => ({ ...old, ...patch })); + + // While the tracks were being added, setStream may have unpublished the source or replaced + // its stream. Record track IDs only where the entry is still the one we started streaming + // (same stream, still no track IDs) — anything else must not be patched (it would resurrect + // an unpublished source or attach the IDs to a different stream), and the tracks we just + // added for it are orphans to unpublish again. + const isStillCurrent = ([id, started]: (typeof results)[number]) => { + const current = sourcesRef.current[id]; + return current !== undefined && current.stream === started.stream && current.trackIds === undefined; + }; + const patch = results.filter(isStillCurrent); + const orphans = results.filter((result) => !isStillCurrent(result)); + + if (patch.length > 0) { + updateSources((old) => ({ ...old, ...Object.fromEntries(patch) })); + } + for (const [, started] of orphans) { + if (started.trackIds) await removeTracks(started.trackIds); + } }; const onDisconnected = () => { - setSources((old) => + updateSources((old) => Object.fromEntries(Object.entries(old).map(([id, source]) => [id, { ...source, trackIds: undefined }])), ); }; - if (peerStatus === "connected") onConnected(); + // addTrack can reject for reasons other than TrackTypeError (e.g. a disconnect mid-add); + // surface it instead of leaving an unhandled rejection. + if (peerStatus === "connected") + onConnected().catch((error) => logger.error("Failed to publish custom sources", error)); fishjamClient.on("disconnected", onDisconnected); return () => { fishjamClient.off("disconnected", onDisconnected); }; - }, [pendingSources, fishjamClient, peerStatus, startStreaming]); + }, [pendingSources, fishjamClient, peerStatus, startStreaming, removeTracks, updateSources, logger]); return { setStream, getSource }; } diff --git a/packages/react-native-vision-camera-source/package.json b/packages/react-native-vision-camera-source/package.json index 46f758ff..06480358 100644 --- a/packages/react-native-vision-camera-source/package.json +++ b/packages/react-native-vision-camera-source/package.json @@ -41,7 +41,6 @@ ], "scripts": { "tsc": "tsc --noEmit", - "test": "vitest run", "build:webrtc-workspace": "yarn workspace @fishjam-cloud/react-native-webrtc run prepare", "build": "yarn build:webrtc-workspace && bob build", "lint": "eslint . --ext .ts,.tsx --fix", @@ -112,8 +111,7 @@ "react-native-vision-camera-worklets": "5.0.10", "react-native-webgpu": "0.5.15", "react-native-worklets": "0.10.2", - "typescript": "^5.8.3", - "vitest": "^3.1.1" + "typescript": "^5.8.3" }, "packageManager": "yarn@4.16.0" } diff --git a/packages/react-native-vision-camera-source/src/frameTimestamp.ts b/packages/react-native-vision-camera-source/src/frameTimestamp.ts index 6d6acb5e..d2f3d29e 100644 --- a/packages/react-native-vision-camera-source/src/frameTimestamp.ts +++ b/packages/react-native-vision-camera-source/src/frameTimestamp.ts @@ -13,6 +13,9 @@ const NANOSECONDS_MAGNITUDE_THRESHOLD = 1e9; const NANOSECONDS_PER_SECOND = 1e9; +/** Default fallback spacing between published frames when one carries no usable timestamp: 30 fps. */ +export const DEFAULT_FRAME_INTERVAL_NANOSECONDS = 33_333_333; + interface FrameTimestampTimeline { /** Absolute timestamp of the first frame, in nanoseconds; the timeline's zero point. */ firstNanoseconds: number | null; diff --git a/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts index 0d6a0ccd..fab8058f 100644 --- a/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts +++ b/packages/react-native-vision-camera-source/src/internal/usePublishedStream.ts @@ -7,8 +7,9 @@ export function usePublishedStream(sourceId: string, stream: MediaStream | null) const { setStream } = useCustomSource(sourceId); useEffect(() => { if (stream == null) return; - // setStream can reject (e.g. two publishes racing over the same source remove the same stale - // track IDs); downgrade to a warning — an unhandled rejection here would take down dev builds. + // Calls for one source are serialized by the manager, but setStream can still reject (e.g. + // a track-removal failure); downgrade to a warning — an unhandled rejection here would take + // down dev builds. setStream(stream).catch((cause: unknown) => { console.warn('usePublishedStream: publishing the stream failed', cause); }); diff --git a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts index bcbe0b3e..45a2e410 100644 --- a/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts +++ b/packages/react-native-vision-camera-source/src/useVisionCameraSource.ts @@ -9,7 +9,11 @@ import { useFrameOutput, } from 'react-native-vision-camera'; -import { createFrameTimestampState, normalizeFrameTimestampNanoseconds } from './frameTimestamp'; +import { + createFrameTimestampState, + DEFAULT_FRAME_INTERVAL_NANOSECONDS, + nextFrameTimestampNanoseconds, +} from './frameTimestamp'; import { usePublishedStream } from './internal/usePublishedStream'; import { rotationDegreesFromOrientation } from './orientation'; @@ -35,6 +39,11 @@ export interface UseVisionCameraSourceOptions extends Partial void; /** Called whenever the camera pipeline drops a frame; forwarded to VisionCamera. */ onFrameDropped?: (reason: FrameDroppedReason) => void; + /** + * Fallback spacing between published frames, in nanoseconds, used only when a camera frame + * carries no usable timestamp. Defaults to 33,333,333 (30 fps). + */ + frameIntervalNanoseconds?: number; } /** Result of {@link useVisionCameraSource}. */ @@ -81,7 +90,13 @@ export function useVisionCameraSource( sourceId: SourceId, options: UseVisionCameraSourceOptions = {}, ): UseVisionCameraSourceResult { - const { enabled = true, onFrame: userOnFrame, onFrameDropped, ...frameOutputOptions } = options; + const { + enabled = true, + onFrame: userOnFrame, + onFrameDropped, + frameIntervalNanoseconds = DEFAULT_FRAME_INTERVAL_NANOSECONDS, + ...frameOutputOptions + } = options; const { track, stream, error } = useManagedForwardTrack(enabled); usePublishedStream(sourceId, stream); @@ -93,15 +108,19 @@ export function useVisionCameraSource( 'worklet'; try { if (track != null) { + // Every frame gets a timeline timestamp (interval-paced when the frame carries no + // usable one): native stamps omitted timestamps — and the value 0, the timeline's + // first frame — with its own clock, and one track must never mix the two domains. + const timestampNanoseconds = Math.max( + 1, + nextFrameTimestampNanoseconds(timestampState, frame.timestamp, frameIntervalNanoseconds), + ); const nativeBuffer = frame.getNativeBuffer(); try { - const timestampNanoseconds = normalizeFrameTimestampNanoseconds(timestampState, frame.timestamp); forwardFrame(track, { nativeBuffer: nativeBuffer.pointer, rotation: rotationDegreesFromOrientation(frame.orientation), - // When the frame carries no usable timestamp, omit it — the native layer then - // stamps the frame with its own monotonic clock. - ...(timestampNanoseconds != null ? { timestampNs: timestampNanoseconds } : {}), + timestampNs: timestampNanoseconds, }); } finally { nativeBuffer.release(); @@ -116,7 +135,7 @@ export function useVisionCameraSource( frame.dispose(); } }; - }, [track, userOnFrame, timestampState]); + }, [track, userOnFrame, timestampState, frameIntervalNanoseconds]); const frameOutput = useFrameOutput({ // 'native' keeps the pipeline copy-free; override via options if a plugin needs 'rgb'/'yuv'. diff --git a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts index d679eb5e..0aca8d75 100644 --- a/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts +++ b/packages/react-native-vision-camera-source/src/webgpu/useVisionCameraWebGpuSource.ts @@ -19,12 +19,15 @@ import { } from 'react-native-vision-camera'; import { type GPUSharedTextureMemory, GPUTextureUsage } from 'react-native-webgpu'; -import { createFrameTimestampState, nextFrameTimestampNanoseconds } from '../frameTimestamp'; +import { + createFrameTimestampState, + DEFAULT_FRAME_INTERVAL_NANOSECONDS, + nextFrameTimestampNanoseconds, +} from '../frameTimestamp'; import { usePublishedStream } from '../internal/usePublishedStream'; import { rotationDegreesFromOrientation } from '../orientation'; const DEFAULT_POOL_SIZE = 3; -const DEFAULT_FRAME_INTERVAL_NANOSECONDS = 33_333_333; // 30 fps fallback cadence /** * Options for {@link useVisionCameraWebGpuSource}. Also accepts every VisionCamera frame-output @@ -167,7 +170,9 @@ export function useVisionCameraWebGpuSource( // [track, device] so a new track or device starts from an empty cache and never touches stale // imports. Imports abandoned by a replaced closure are released by the frame runtime's GC; the // deterministic alternative (import + destroy every frame) costs an import per frame — switch - // to it if leak measurements ever demand. + // to it if leak measurements ever demand. Note that every recreation of `handleFrame` copies + // the pristine JS-side object into the new worklet closure and re-imports the pool — which is + // why the option docs insist on a stable `onFrame` identity. const workletState = useMemo(() => { // track/device are not read here — the `void`s mark them as intentional reset-only deps. void track; @@ -264,6 +269,8 @@ export function useVisionCameraWebGpuSource( accessResult = imported.memory.endAccess(imported.texture); } + // endAccess returns one fence per queue that touched the memory; this access + // scope only ever submits once on the one device queue, so [0] is the whole set. const fenceState = accessResult.fences[0]; const timestampNanoseconds = nextFrameTimestampNanoseconds( timestampState, diff --git a/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts b/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts deleted file mode 100644 index 0a0d489a..00000000 --- a/packages/react-native-vision-camera-source/tests/frameTimestamp.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -import { - createFrameTimestampState, - type FrameTimestampState, - nextFrameTimestampNanoseconds, - normalizeFrameTimestampNanoseconds, -} from '../src/frameTimestamp'; - -// In-memory stand-in for react-native-worklets' Synchronizable (the real implementation needs a -// native runtime; vitest hoists this mock above the imports). Faithful to the surface -// frameTimestamp uses: getBlocking + setBlocking. -vi.mock('react-native-worklets', () => ({ - createSynchronizable: (initialValue: TValue) => { - let value = initialValue; - return { - getBlocking: () => value, - setBlocking: (next: TValue | ((previous: TValue) => TValue)) => { - value = typeof next === 'function' ? (next as (previous: TValue) => TValue)(value) : next; - }, - }; - }, -})); - -const NANOSECONDS_PER_SECOND = 1e9; - -let state: FrameTimestampState; -beforeEach(() => { - state = createFrameTimestampState(); -}); - -describe('normalizeFrameTimestampNanoseconds', () => { - it('rejects unusable timestamps (invalid frames report 0)', () => { - expect(normalizeFrameTimestampNanoseconds(state, 0)).toBeNull(); - expect(normalizeFrameTimestampNanoseconds(state, -5)).toBeNull(); - expect(normalizeFrameTimestampNanoseconds(state, Number.NaN)).toBeNull(); - }); - - it('treats iOS CMTime seconds as seconds and preserves inter-frame gaps in nanoseconds', () => { - // Typical iOS values: seconds since boot, ~1e5–1e6. - const firstSeconds = 123456.5; - const secondSeconds = 123456.5334; - expect(normalizeFrameTimestampNanoseconds(state, firstSeconds)).toBe(0); - expect(normalizeFrameTimestampNanoseconds(state, secondSeconds)).toBe( - secondSeconds * NANOSECONDS_PER_SECOND - firstSeconds * NANOSECONDS_PER_SECOND, - ); - }); - - it('treats Android CameraX nanoseconds as nanoseconds after long uptime', () => { - // ~28 hours of uptime, 33 ms apart. - const firstNanoseconds = 1e14; - expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds)).toBe(0); - expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds + 33_000_000)).toBe(33_000_000); - }); - - it('treats Android nanoseconds as nanoseconds on a recently booted device (regression)', () => { - // 2 minutes of uptime: 1.2e11 ns. The old 1e12 threshold misclassified this as seconds and - // scaled a 33 ms gap into ~380 days of timeline. - const firstNanoseconds = 1.2e11; - expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds)).toBe(0); - expect(normalizeFrameTimestampNanoseconds(state, firstNanoseconds + 33_000_000)).toBe(33_000_000); - }); - - it('stays continuous when an Android stream crosses a magnitude boundary mid-stream', () => { - // Frames straddling 1e12 ns (~16.7 min of uptime) — the old threshold flipped units here. - const beforeBoundary = 1e12 - 16_000_000; - const afterBoundary = 1e12 + 17_000_000; - expect(normalizeFrameTimestampNanoseconds(state, beforeBoundary)).toBe(0); - expect(normalizeFrameTimestampNanoseconds(state, afterBoundary)).toBe(33_000_000); - }); - - it('rejects timestamps that do not advance the timeline', () => { - expect(normalizeFrameTimestampNanoseconds(state, 1e14)).toBe(0); - expect(normalizeFrameTimestampNanoseconds(state, 1e14)).toBeNull(); - expect(normalizeFrameTimestampNanoseconds(state, 1e14 - 1_000_000)).toBeNull(); - // A later frame that does advance is accepted again. - expect(normalizeFrameTimestampNanoseconds(state, 1e14 + 1_000_000)).toBe(1_000_000); - }); -}); - -describe('nextFrameTimestampNanoseconds', () => { - const frameIntervalNanoseconds = 33_333_333; - - it('passes real timestamps through', () => { - expect(nextFrameTimestampNanoseconds(state, 1e14, frameIntervalNanoseconds)).toBe(0); - expect(nextFrameTimestampNanoseconds(state, 1e14 + 40_000_000, frameIntervalNanoseconds)).toBe(40_000_000); - }); - - it('paces by the frame interval when frames carry no timestamp', () => { - expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(0); - expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(frameIntervalNanoseconds); - expect(nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds)).toBe(2 * frameIntervalNanoseconds); - }); - - it('stays monotonic when real timestamps appear after interval-paced frames (documented quirk)', () => { - const paced = nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds); - const pacedAgain = nextFrameTimestampNanoseconds(state, 0, frameIntervalNanoseconds); - // First real timestamp becomes the baseline (offset 0), which is behind the paced timeline — - // so it falls back to interval pacing rather than jumping backwards. - const afterRealTimestamp = nextFrameTimestampNanoseconds(state, 1e14, frameIntervalNanoseconds); - expect(paced).toBe(0); - expect(pacedAgain).toBeGreaterThan(paced); - expect(afterRealTimestamp).toBeGreaterThan(pacedAgain); - }); -}); diff --git a/yarn.lock b/yarn.lock index 9b3f066a..060692a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5398,7 +5398,6 @@ __metadata: react-native-webgpu: "npm:0.5.15" react-native-worklets: "npm:0.10.2" typescript: "npm:^5.8.3" - vitest: "npm:^3.1.1" peerDependencies: "@fishjam-cloud/react-native-client": "workspace:*" "@fishjam-cloud/react-native-webrtc": "workspace:*"