FCE-3602: Create mobile custom source high level API

.github/workflows/release.yaml

@@ -91,3 +91,52 @@ jobs:
         working-directory: packages/mobile-client
       - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }}
         working-directory: packages/mobile-client
+
+  publish-react-native-custom-video-source:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+          registry-url: "https://registry.npmjs.org"
+      - run: corepack enable
+      - run: yarn
+      - run: yarn workspace @fishjam-cloud/react-native-custom-video-source run check:webrtc-published
+      - run: yarn build
+      - run: yarn pack --out package.tgz
+        working-directory: packages/react-native-custom-video-source
+      - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }}
+        working-directory: packages/react-native-custom-video-source
+
+  publish-react-native-vision-camera-source:
+    # vision-camera-source depends on custom-video-source at the same version, so it must not
+    # reach npm unless that dependency's publish succeeded.
+    needs: publish-react-native-custom-video-source
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      # Setup .npmrc file to publish to npm
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+          registry-url: "https://registry.npmjs.org"
+      - run: corepack enable
+      - run: yarn
+      - run: yarn workspace @fishjam-cloud/react-native-vision-camera-source run check:webrtc-published
+      - run: yarn build
+      - run: yarn pack --out package.tgz
+        working-directory: packages/react-native-vision-camera-source
+      - run: npm publish package.tgz --provenance --access public ${{ github.event.release.prerelease && '--tag rc' || '' }}
+        working-directory: packages/react-native-vision-camera-source

package.json

@@ -9,6 +9,8 @@
     "packages/react-client",
     "packages/webrtc-client",
     "packages/mobile-client",
+    "packages/react-native-custom-video-source",
+    "packages/react-native-vision-camera-source",
     "packages/react-native-webrtc",
     "examples/react-client/*",
     "examples/mobile-client/*",
@@ -36,7 +38,7 @@
   },
   "resolutions": {
     "jest-snapshot-prettier": "npm:prettier@^3",
-    "@types/react": "19.1.17"
+    "@types/react": "19.2.14"
   },
   "devDependencies": {
     "@fishjam-cloud/protobufs": "workspace:^",

packages/react-client/src/hooks/internal/useCustomSourceManager.ts

@@ -1,5 +1,5 @@
 import { type FishjamClient, type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
 import type { CustomSourceState, CustomSourceTracks } from "../../types/internal";
 import type { PeerStatus } from "../../types/public";
@@ -21,6 +21,18 @@ export function useCustomSourceManager({
   logger,
 }: CustomSourceManagerProps): CustomSourceManager {
   const [sources, setSources] = useState<Record<string, CustomSourceState>>({});
+  // setStream reads the current sources synchronously (to diff against the old stream and to
+  // remove stale tracks), but must stay referentially stable so consumers can safely depend on
+  // it in effects. Reading through a ref keeps it out of the useCallback deps.
+  const sourcesRef = useRef(sources);
+  sourcesRef.current = sources;
+
+  // Replacing a source's stream issues two setStream calls in the same tick (unpublish the old,
+  // publish the new). Run concurrently they would read the same snapshot and race to remove the
+  // same track IDs — and when the publish call loses that race, the new stream is never
+  // published. This chain serializes the calls instead.
+  const setStreamQueue = useRef(Promise.resolve());
+
   const pendingSources = useMemo(
     () => Object.entries(sources).filter(([_, source]) => source.trackIds === undefined),
     [sources],
@@ -86,47 +98,87 @@ export function useCustomSourceManager({
 
   const getSource = useCallback((sourceId: string) => sources[sourceId], [sources]);
 
-  const setStream = useCallback(
+  // Updates both the state (what consumers render) and the ref (what queued setStream calls
+  // read synchronously, before React re-renders).
+  const updateSources = useCallback(
+    (update: (old: Record<string, CustomSourceState>) => Record<string, CustomSourceState>) => {
+      sourcesRef.current = update(sourcesRef.current);
+      setSources(update);
+    },
+    [],
+  );
+
+  const applySetStream = useCallback(
     async (sourceId: string, stream: MediaStream | null) => {
-      const oldSource = sources[sourceId];
+      const oldSource = sourcesRef.current[sourceId];
       if (stream === oldSource?.stream) return;
 
       if (oldSource?.trackIds) await removeTracks(oldSource.trackIds);
 
       if (stream !== null) {
-        setSources((old) => ({ ...old, [sourceId]: { stream } }));
-        return;
+        updateSources((old) => ({ ...old, [sourceId]: { stream } }));
+      } else if (oldSource) {
+        updateSources((old) => Object.fromEntries(Object.entries(old).filter(([id]) => id !== sourceId)));
       }
-      if (!oldSource) return;
+    },
+    [removeTracks, updateSources],
+  );
 
-      setSources((old) => Object.fromEntries(Object.entries(old).filter(([id, _]) => id !== sourceId)));
+  const setStream = useCallback(
+    (sourceId: string, stream: MediaStream | null) => {
+      const run = setStreamQueue.current.then(() => applySetStream(sourceId, stream));
+      // Chain a never-rejecting link so one failed call cannot poison the queue; the caller
+      // still observes failures through the returned promise.
+      setStreamQueue.current = run.catch(() => undefined);
+      return run;
     },
-    [sources, removeTracks],
+    [applySetStream],
   );
 
   useEffect(() => {
     const onConnected = async () => {
       if (pendingSources.length === 0) return;
 
-      const patch = Object.fromEntries(
-        await Promise.all(pendingSources.map(async ([id, source]) => [id, await startStreaming(source)] as const)),
+      const results = await Promise.all(
+        pendingSources.map(async ([id, source]) => [id, await startStreaming(source)] as const),
       );
-      setSources((old) => ({ ...old, ...patch }));
+
+      // While the tracks were being added, setStream may have unpublished the source or replaced
+      // its stream. Record track IDs only where the entry is still the one we started streaming
+      // (same stream, still no track IDs) — anything else must not be patched (it would resurrect
+      // an unpublished source or attach the IDs to a different stream), and the tracks we just
+      // added for it are orphans to unpublish again.
+      const isStillCurrent = ([id, started]: (typeof results)[number]) => {
+        const current = sourcesRef.current[id];
+        return current !== undefined && current.stream === started.stream && current.trackIds === undefined;
+      };
+      const patch = results.filter(isStillCurrent);
+      const orphans = results.filter((result) => !isStillCurrent(result));
+
+      if (patch.length > 0) {
+        updateSources((old) => ({ ...old, ...Object.fromEntries(patch) }));
+      }
+      for (const [, started] of orphans) {
+        if (started.trackIds) await removeTracks(started.trackIds);
+      }
     };
 
     const onDisconnected = () => {
-      setSources((old) =>
+      updateSources((old) =>
         Object.fromEntries(Object.entries(old).map(([id, source]) => [id, { ...source, trackIds: undefined }])),
       );
     };
 
-    if (peerStatus === "connected") onConnected();
+    // addTrack can reject for reasons other than TrackTypeError (e.g. a disconnect mid-add);
+    // surface it instead of leaving an unhandled rejection.
+    if (peerStatus === "connected")
+      onConnected().catch((error) => logger.error("Failed to publish custom sources", error));
 
     fishjamClient.on("disconnected", onDisconnected);
     return () => {
       fishjamClient.off("disconnected", onDisconnected);
     };
-  }, [pendingSources, fishjamClient, peerStatus, startStreaming]);
+  }, [pendingSources, fishjamClient, peerStatus, startStreaming, removeTracks, updateSources, logger]);
 
   return { setStream, getSource };
 }

packages/react-client/src/hooks/useCustomSource.ts

@@ -4,6 +4,8 @@ import { CustomSourceContext } from "../contexts/customSource";
 
 /**
  * This hook can register/deregister a custom MediaStream with Fishjam.
+ *
+ * @param sourceId - Stable id identifying this custom source.
  * @group Hooks
  */
 export function useCustomSource<T extends string>(sourceId: T) {

packages/react-native-custom-video-source/.eslintignore

@@ -0,0 +1,3 @@
+dist
+build
+coverage

packages/react-native-custom-video-source/.eslintrc

@@ -0,0 +1,13 @@
+{
+  "extends": ["../../.eslintrc.js", "expo", "prettier"],
+  "plugins": ["prettier"],
+  "ignorePatterns": ["lib", "/dist/*"],
+  "rules": {
+    "prettier/prettier": "error"
+  },
+  "settings": {
+    "import/resolver": {
+      "typescript": true
+    }
+  }
+}

packages/react-native-custom-video-source/.prettierrc

@@ -0,0 +1,9 @@
+{
+  "bracketSameLine": true,
+  "printWidth": 120,
+  "quoteProps": "consistent",
+  "singleQuote": true,
+  "tabWidth": 2,
+  "trailingComma": "all",
+  "useTabs": false
+}

packages/react-native-custom-video-source/README.md

@@ -0,0 +1,70 @@
+# `@fishjam-cloud/react-native-custom-video-source`
+
+Publish your own video frames to Fishjam from **any** frame source. This is the generic layer
+under [`@fishjam-cloud/react-native-vision-camera-source`](../react-native-vision-camera-source) —
+use that package if your source is [VisionCamera](https://react-native-vision-camera.com); use this
+one directly for any other source (a native ML pipeline, a compositor, your own renderer).
+
+There are two modes, picked by **how you produce frames**:
+
+- **Forwarding** — you already have finished native buffers. `useManagedForwardTrack()` creates and
+  owns the track; hand each buffer pointer to `forwardFrame` from `@fishjam-cloud/react-native-webrtc`.
+- **Render-target (pooled)** — you render the frames yourself. `useManagedPooledTrack()` allocates a
+  surface pool; draw into it and hand each frame back with `pushFrame`. The
+  [`/webgpu`](#webgpu-camera-toolkit) entry point provides a WebGPU camera-rendering toolkit for this
+  mode.
+
+Each hook owns the track's async lifecycle — creation, the published `stream`, and teardown — so you
+only feed frames. Must be used under `FishjamProvider` from `@fishjam-cloud/react-native-client`.
+
+## Prerequisites
+
+- `@fishjam-cloud/react-native-webrtc` and `@fishjam-cloud/react-native-client` (with your app wrapped
+  in `FishjamProvider`)
+- New Architecture (custom video tracks require it)
+- For the `/webgpu` entry only: `react-native-webgpu` ≥ 0.5.15, plus `unplugin-typegpu` in your app's
+  Babel config (the shaders are authored in [TypeGPU](https://docs.swmansion.com/TypeGPU/)), and
+  **iOS 17+** for the Metal external-texture camera-import path.
+
+## Forward finished buffers
+
+```tsx
+import { useManagedForwardTrack } from '@fishjam-cloud/react-native-custom-video-source';
+import { forwardFrame } from '@fishjam-cloud/react-native-webrtc';
+import { useCustomSource } from '@fishjam-cloud/react-native-client';
+
+const { track, stream } = useManagedForwardTrack(/* enabled */ true);
+// publish `stream` (e.g. via useCustomSource), then per frame:
+forwardFrame(track, { nativeBuffer /* bigint CVPixelBufferRef / AHardwareBuffer* */ });
+```
+
+## Render your own frames
+
+```tsx
+import { useManagedPooledTrack } from '@fishjam-cloud/react-native-custom-video-source';
+import { pushFrame } from '@fishjam-cloud/react-native-webrtc';
+
+const { track, bufferDescriptors } = useManagedPooledTrack(true, 720, 1280, /* poolSize */ 3);
+// import bufferDescriptors[i].surfaceHandle into your GPU, render into slot i, then:
+pushFrame(track, { bufferIndex: i, timestampNs });
+```
+
+## WebGPU camera toolkit
+
+`@fishjam-cloud/react-native-custom-video-source/webgpu` samples the live camera and helps you render
+into the pooled surfaces:
+
+- `createCameraShaderBindings` / `sampleCamera` — sample the camera from your own shaders, YUV decode
+  handled per platform
+- `createCameraPassthroughPipeline` / `encodeCameraPassthrough` — a ready-made camera→output pass
+- `useCameraWebGpuDevice` — the shared, camera-import-capable `GPUDevice`
+- `computeAspectFillCrop` / `computeSquareCrop` / `packFrameCropParams` — crop + orientation helpers
+
+> Verify camera + WebGPU behavior on **physical devices** — the iOS Simulator cannot import the
+> camera's YUV textures.
+
+## Development
+
+Part of the [web-client-sdk](https://github.com/fishjam-cloud/web-client-sdk) monorepo. `yarn build`
+compiles `src/` to `dist/` with `react-native-builder-bob` (Babel + `unplugin-typegpu` for the TGSL
+shaders, `tsc` for type definitions).

packages/react-native-custom-video-source/babel.config.js

@@ -0,0 +1,16 @@
+// Build-time Babel config, used only by react-native-builder-bob when compiling this package's
+// own source to `dist` (bob is configured with `configFile: true`, so it defers entirely to this
+// config instead of its default presets). It is never shipped — the published tarball is `dist`
+// only — so it does not affect how consumers' Metro transpiles the built output.
+//
+// - `@babel/preset-typescript` strips the TypeScript types. It preserves `'worklet'` string
+//   directives, which must survive into `dist` so the consumer app's react-native-worklets Babel
+//   plugin can pick them up. We deliberately do NOT run the worklets plugin here — that is the
+//   consumer's job at app-build time.
+// - `unplugin-typegpu/babel` transforms TGSL (`tgpu.fn(... )(js => ...)`) function bodies into the
+//   tinyest AST that TypeGPU resolves to WGSL at runtime. Without it, TGSL functions throw on
+//   `tgpu.resolve`.
+module.exports = {
+  presets: [['@babel/preset-typescript', { onlyRemoveTypeImports: true }]],
+  plugins: [['unplugin-typegpu/babel', { forceTgpuAlias: 'tgpu' }]],
+};