diff --git a/README.md b/README.md index 8f5d9b249..ce44e5236 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@

  • Cross Platform (iOS and Android)

  • Optimized for performance and high photo capture rate

  • QR / Barcode scanning support

    • +

  • Face detection support

  • Camera preview support in iOS simulator

    • @@ -170,6 +171,45 @@ Additionally, the Camera can be used for barcode scanning /> ``` +#### Face Detection + +Detect faces in real time. iOS uses Apple Vision; Android uses Google ML Kit. + +> **Android requires a Google Play Store device** + +```tsx + { + // event.nativeEvent.faces: FaceData[] + // each face: { id, yaw, pitch, roll, boundsX, boundsY, boundsWidth, boundsHeight } + }} + // Android only — track MLKit face module download progress + onFaceDetectionInstallStatus={(event) => { + // event.nativeEvent.state: FaceDetectionInstallState + // 'pending' | 'downloading' | 'installing' | 'ready' | 'failed' | 'unavailable' + }} +/> +``` + +##### Android: pre-download the ML Kit model + +This library depends on the unbundled `play-services-mlkit-face-detection` variant — the model is downloaded by Google Play Services rather than bundled in the APK. By default the download happens on first use, and `onFaceDetectionInstallStatus` reports the progress. + +To have Play Services [pre-download the model in the background after the app is installed](https://developers.google.com/ml-kit/tips/installation-paths#how_to_download_models), add this to your app's `AndroidManifest.xml`: + +```xml + + + +``` + +> **Note:** Requires Google Play Services on the device. On devices without Google Play Services, `onFaceDetectionInstallStatus` fires once with `'unavailable'`. + ### Camera Props (Optional) | Props | Type | Description | @@ -197,7 +237,7 @@ Additionally, the Camera can be used for barcode scanning | `resizeMode` | `'cover' / 'contain'` | Determines the scaling and cropping behavior of content within the view. `cover` (resizeAspectFill on iOS) scales the content to fill the view completely, potentially cropping content if its aspect ratio differs from the view. `contain` (resizeAspect on iOS) scales the content to fit within the view's bounds without cropping, ensuring all content is visible but may introduce letterboxing. Default behavior depends on the specific use case. | | `scanThrottleDelay` | `number` | Duration between scan detection in milliseconds. Default 2000 (2s) | | `maxPhotoQualityPrioritization` | `'balanced'` / `'quality'` / `'speed'` | [iOS 13 and newer](https://developer.apple.com/documentation/avfoundation/avcapturephotooutput/3182995-maxphotoqualityprioritization). `'speed'` provides a 60-80% median capture time reduction vs 'quality' setting. Tested on iPhone 6S Max (66% faster) and iPhone 15 Pro Max (76% faster!). Default `balanced` | -| `iOsDeferredStart` | `boolean` | iOS 26+ only. Enables `AVCaptureOutput.deferredStartEnabled` when supported to get the preview visible faster. Default `true`. When enabled, the first capture can be delayed by a few hundred milliseconds. Ignored on Android and on older iOS versions. | +| `iOsDeferredStart` | `boolean` | iOS 26+ only. Enables `AVCaptureOutput.deferredStartEnabled` when supported to get the preview visible faster. Default `true`. When enabled, the first capture can be delayed by a few hundred milliseconds. Ignored on Android and on older iOS versions. | | `onCaptureButtonPressIn` | Function | Callback when iPhone capture button is pressed in or Android volume or camera button is pressed in. Ex: `onCaptureButtonPressIn={() => console.log("volume button pressed in")}` | | `onCaptureButtonPressOut` | Function | Callback when iPhone capture button is released or Android volume or camera button is released. Ex: `onCaptureButtonPressOut={() => console.log("volume button released")}` | | **Barcode only** | @@ -207,6 +247,11 @@ Additionally, the Camera can be used for barcode scanning | `laserColor` | Color | Color of barcode scanner laser visualization. Default: `red` | | `frameColor` | Color | Color of barcode scanner frame visualization. Default: `yellow` | | `onReadCode` | Function | Callback when scanner successfully reads barcode. Returned event contains `codeStringValue`. Default: `null`. Ex: `onReadCode={(event) => console.log(event.nativeEvent.codeStringValue)}` | +| **Face detection** | +| `faceDetectionEnabled` | `boolean` | Enable real-time face detection. Default: `false` | +| `faceDetectionThrottleMs` | `number` | Minimum milliseconds between `onFaceDetected` emits. Default: `100` | +| `onFaceDetected` | Function | Callback while face detection is active, with one entry per detected face (empty array if none). | +| `onFaceDetectionInstallStatus` | Function | **Android only.** Callback while the MLKit face detection module is being downloaded by Google Play Services. See [Android: pre-download the ML Kit model](#android-pre-download-the-ml-kit-model) above to skip the first-run download. | ### Imperative API diff --git a/android/build.gradle b/android/build.gradle index 6cc073301..145733b84 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -67,6 +67,7 @@ dependencies { // implementation "androidx.camera:camera-extensions:${camerax_version}" implementation 'com.google.mlkit:barcode-scanning:17.3.0' + implementation 'com.google.android.gms:play-services-mlkit-face-detection:17.1.0' } repositories { mavenCentral() diff --git a/android/src/main/java/com/rncamerakit/CKCamera.kt b/android/src/main/java/com/rncamerakit/CKCamera.kt index 928c5e361..bf8d9e7f4 100644 --- a/android/src/main/java/com/rncamerakit/CKCamera.kt +++ b/android/src/main/java/com/rncamerakit/CKCamera.kt @@ -45,6 +45,8 @@ import android.graphics.Rect import android.graphics.RectF import android.util.Size import com.facebook.react.uimanager.UIManagerHelper +import com.google.android.gms.tasks.Task +import com.google.android.gms.tasks.Tasks import com.google.mlkit.vision.barcode.common.Barcode import com.rncamerakit.events.* @@ -83,6 +85,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs private var preview: Preview? = null private var imageCapture: ImageCapture? = null private var imageAnalyzer: ImageAnalysis? = null + private var faceAnalyzer: FaceAnalyzer? = null private var orientationListener: OrientationEventListener? = null private var viewFinder: PreviewView = PreviewView(context) private var rectOverlay: RectOverlay = RectOverlay(context) @@ -112,6 +115,10 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs private var barcodeFrameSize: Size? = null private var allowedBarcodeTypes: Array? = null + // Face detection props + private var faceDetectionEnabled: Boolean = false + private var faceDetectionThrottleMs: Long = FaceAnalyzer.DEFAULT_THROTTLE_MS + private fun getActivity() : Activity { return currentContext.currentActivity!! } @@ -142,11 +149,21 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs override fun onDetachedFromWindow() { super.onDetachedFromWindow() + faceAnalyzer?.close() + faceAnalyzer = null cameraExecutor.shutdown() orientationListener?.disable() cameraProvider?.unbindAll() } + override fun onWindowFocusChanged(hasWindowFocus: Boolean) { + super.onWindowFocusChanged(hasWindowFocus) + if (hasWindowFocus && cameraProvider == null && + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) { + viewFinder.post { setupCamera() } + } + } + override fun dispatchKeyEvent(event: KeyEvent?): Boolean { val keyCode = event?.getKeyCode() val action = event?.getAction() @@ -341,8 +358,11 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs val useCases = mutableListOf(preview, imageCapture) - if (scanBarcode) { - val analyzer = QRCodeAnalyzer({ barcodes, imageSize -> + faceAnalyzer?.close() + faceAnalyzer = null + + val barcodeAnalyzer: QRCodeAnalyzer? = if (scanBarcode) { + QRCodeAnalyzer({ barcodes, imageSize -> if (barcodes.isEmpty()) return@QRCodeAnalyzer // 1. Filter by allowed barcode formats @@ -390,7 +410,27 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs onBarcodeRead(filteredBarcodes) } }, scanThrottleDelay) - imageAnalyzer!!.setAnalyzer(cameraExecutor, analyzer) + } else null + + faceAnalyzer = if (faceDetectionEnabled) { + FaceAnalyzer( + faceDetectionThrottleMs, + context, + viewFinder, + { state -> onFaceDetectionInstallStatus(state) }, + { payloads -> onFaceDetected(payloads) } + ) + } else null + + val activeFaceAnalyzer = faceAnalyzer + if (barcodeAnalyzer != null || activeFaceAnalyzer != null) { + imageAnalyzer!!.setAnalyzer(cameraExecutor) { image -> + val tasks = mutableListOf>() + barcodeAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) } + activeFaceAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) } + if (tasks.isEmpty()) image.close() + else Tasks.whenAllComplete(tasks).addOnCompleteListener { image.close() } + } useCases.add(imageAnalyzer) } @@ -547,6 +587,27 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs ?.dispatchEvent(ReadCodeEvent(surfaceId, id, barcodes.first().rawValue, codeFormat.code)) } + private fun onFaceDetected(payloads: List) { + // CameraX auto-mirrors the front-camera Preview but not ImageAnalysis, + // so flip X here to keep bounds in preview-space for JS consumers. + val mirrored = if (lensType == CameraSelector.LENS_FACING_FRONT) { + payloads.map { it.copy(boundsX = 1.0 - it.boundsX - it.boundsWidth) } + } else { + payloads + } + val surfaceId = UIManagerHelper.getSurfaceId(currentContext) + UIManagerHelper + .getEventDispatcherForReactTag(currentContext, id) + ?.dispatchEvent(FaceDetectedEvent(surfaceId, id, mirrored)) + } + + private fun onFaceDetectionInstallStatus(state: String) { + val surfaceId = UIManagerHelper.getSurfaceId(currentContext) + UIManagerHelper + .getEventDispatcherForReactTag(currentContext, id) + ?.dispatchEvent(FaceDetectionInstallStatusEvent(surfaceId, id, state)) + } + private fun onOrientationChange(orientation: Int) { val remappedOrientation = when (orientation) { Surface.ROTATION_0 -> RNCameraKitModule.PORTRAIT @@ -674,6 +735,19 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs if (restartCamera) bindCameraUseCases() } + fun setFaceDetectionEnabled(enabled: Boolean) { + val restartCamera = enabled != faceDetectionEnabled + faceDetectionEnabled = enabled + if (restartCamera) bindCameraUseCases() + } + + fun setFaceDetectionThrottleMs(throttleMs: Int) { + val newThrottle = if (throttleMs < 0) FaceAnalyzer.DEFAULT_THROTTLE_MS else throttleMs.toLong() + if (faceDetectionThrottleMs == newThrottle) return + faceDetectionThrottleMs = newThrottle + faceAnalyzer?.throttleMs = newThrottle + } + fun setScanThrottleDelay(delayMs: Int) { val newDelay = if (delayMs < 0) 2000L else delayMs.toLong() val restartCamera = scanThrottleDelay != newDelay && scanBarcode diff --git a/android/src/main/java/com/rncamerakit/FaceAnalyzer.kt b/android/src/main/java/com/rncamerakit/FaceAnalyzer.kt new file mode 100644 index 000000000..058ef88d6 --- /dev/null +++ b/android/src/main/java/com/rncamerakit/FaceAnalyzer.kt @@ -0,0 +1,218 @@ +package com.rncamerakit + +import android.annotation.SuppressLint +import android.content.Context +import android.util.Log +import androidx.camera.core.ExperimentalGetImage +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import androidx.camera.view.PreviewView +import com.google.android.gms.common.ConnectionResult +import com.google.android.gms.common.GoogleApiAvailability +import com.google.android.gms.common.moduleinstall.InstallStatusListener +import com.google.android.gms.common.moduleinstall.ModuleInstall +import com.google.android.gms.common.moduleinstall.ModuleInstallClient +import com.google.android.gms.common.moduleinstall.ModuleInstallRequest +import com.google.android.gms.common.moduleinstall.ModuleInstallStatusUpdate +import com.google.android.gms.tasks.Task +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetector +import com.google.mlkit.vision.face.FaceDetectorOptions +import kotlin.math.max + +data class FacePayload( + val id: Int, + val yaw: Double, + val pitch: Double, + val roll: Double, + val boundsX: Double, + val boundsY: Double, + val boundsWidth: Double, + val boundsHeight: Double, +) + +class FaceAnalyzer( + @Volatile var throttleMs: Long, + context: Context, + private val previewView: PreviewView, + private val onInstallStatus: (state: String) -> Unit, + private val onFaceDetected: (payloads: List) -> Unit +) : ImageAnalysis.Analyzer { + + @Volatile private var detector: FaceDetector? = null + @Volatile private var closed = false + @Volatile private var moduleClient: ModuleInstallClient? = null + @Volatile private var installListener: InstallStatusListener? = null + + private var lastEmitMs = 0L + private var nextLocalId: Int = -1 + + init { + ensureModuleAndCreateDetector(context.applicationContext) + } + + private fun createDetector() = FaceDetection.getClient( + FaceDetectorOptions.Builder() + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_NONE) + .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_NONE) + .setMinFaceSize(MIN_FACE_SIZE) + .enableTracking() + .build() + ) + + private fun setDetectorIfAlive(d: FaceDetector) { + if (closed) d.close() else detector = d + } + + private fun ensureModuleAndCreateDetector(context: Context) { + val status = GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context) + if (status != ConnectionResult.SUCCESS) { + Log.w(TAG, "Google Play Services unavailable (status=$status); face detection disabled.") + onInstallStatus("unavailable") + return + } + + val newDetector = createDetector() + val client = ModuleInstall.getClient(context).also { moduleClient = it } + + client.areModulesAvailable(newDetector) + .addOnSuccessListener { response -> + if (response.areModulesAvailable()) { + setDetectorIfAlive(newDetector) + onInstallStatus("ready") + } else { + onInstallStatus("pending") + requestInstall(client, newDetector) + } + } + .addOnFailureListener { + onInstallStatus("pending") + requestInstall(client, newDetector) + } + } + + private fun requestInstall(client: ModuleInstallClient, newDetector: FaceDetector) { + val listener = InstallStatusListener { update -> + when (update.installState) { + ModuleInstallStatusUpdate.InstallState.STATE_DOWNLOADING -> + onInstallStatus("downloading") + ModuleInstallStatusUpdate.InstallState.STATE_INSTALLING -> + onInstallStatus("installing") + ModuleInstallStatusUpdate.InstallState.STATE_COMPLETED -> { + setDetectorIfAlive(newDetector) + onInstallStatus("ready") + unregisterInstallListener() + } + ModuleInstallStatusUpdate.InstallState.STATE_FAILED, + ModuleInstallStatusUpdate.InstallState.STATE_CANCELED -> { + newDetector.close() + Log.w(TAG, "MLKit face module install ended in state=${update.installState}") + onInstallStatus("failed") + unregisterInstallListener() + } + else -> {} + } + } + installListener = listener + + val request = ModuleInstallRequest.newBuilder() + .addApi(newDetector) + .setListener(listener) + .build() + + client.installModules(request) + .addOnSuccessListener { response -> + if (response.areModulesAlreadyInstalled()) { + setDetectorIfAlive(newDetector) + onInstallStatus("ready") + unregisterInstallListener() + } + } + .addOnFailureListener { e -> + newDetector.close() + Log.w(TAG, "MLKit face module install request failed: ${e.message}") + onInstallStatus("failed") + unregisterInstallListener() + } + } + + private fun unregisterInstallListener() { + val l = installListener ?: return + installListener = null + moduleClient?.unregisterListener(l) + } + + @SuppressLint("UnsafeExperimentalUsageError") + @ExperimentalGetImage + fun analyzeWithoutClosing(image: ImageProxy): Task<*>? { + val det = detector ?: return null + val mediaImage = image.image ?: return null + + val now = System.currentTimeMillis() + if (now - lastEmitMs < throttleMs) { + return null + } + lastEmitMs = now + + val rotation = image.imageInfo.rotationDegrees + val inputImage = InputImage.fromMediaImage(mediaImage, rotation) + val width = if (rotation == 90 || rotation == 270) image.height else image.width + val height = if (rotation == 90 || rotation == 270) image.width else image.height + + return det.process(inputImage) + .addOnSuccessListener { faces -> dispatch(faces, width, height) } + } + + @SuppressLint("UnsafeExperimentalUsageError") + @ExperimentalGetImage + override fun analyze(image: ImageProxy) { + val task = analyzeWithoutClosing(image) + if (task == null) { + image.close() + return + } + task.addOnCompleteListener { image.close() } + } + + private fun dispatch(faces: List, imgWidth: Int, imgHeight: Int) { + val viewW = previewView.width.toFloat() + val viewH = previewView.height.toFloat() + if (viewW <= 0f || viewH <= 0f) return + val srcW = imgWidth.toFloat().coerceAtLeast(1f) + val srcH = imgHeight.toFloat().coerceAtLeast(1f) + val scale = max(viewW / srcW, viewH / srcH) + val offsetX = (viewW - srcW * scale) / 2f + val offsetY = (viewH - srcH * scale) / 2f + + val payloads = faces.map { face -> + val box = face.boundingBox + FacePayload( + id = face.trackingId ?: nextLocalId.also { nextLocalId-- }, + yaw = face.headEulerAngleY.toDouble(), + pitch = face.headEulerAngleX.toDouble(), + roll = face.headEulerAngleZ.toDouble(), + boundsX = ((offsetX + box.left * scale) / viewW).toDouble(), + boundsY = ((offsetY + box.top * scale) / viewH).toDouble(), + boundsWidth = (box.width() * scale / viewW).toDouble(), + boundsHeight = (box.height() * scale / viewH).toDouble(), + ) + } + onFaceDetected(payloads) + } + + fun close() { + closed = true + unregisterInstallListener() + detector?.close() + detector = null + } + + companion object { + private const val TAG = "FaceAnalyzer" + private const val MIN_FACE_SIZE = 0.15f + const val DEFAULT_THROTTLE_MS = 100L + } +} diff --git a/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt b/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt index dd3bfc665..9faf67bd7 100644 --- a/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt +++ b/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt @@ -5,6 +5,7 @@ import android.util.Size import androidx.camera.core.ExperimentalGetImage import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageProxy +import com.google.android.gms.tasks.Task import com.google.mlkit.vision.barcode.BarcodeScanning import com.google.mlkit.vision.barcode.common.Barcode import com.google.mlkit.vision.common.InputImage @@ -15,15 +16,16 @@ class QRCodeAnalyzer ( ) : ImageAnalysis.Analyzer { // Time in milliseconds of the last time we dispatched detected barcodes private var lastBarcodeDetectedTime: Long = 0L + @SuppressLint("UnsafeExperimentalUsageError") @ExperimentalGetImage - override fun analyze(image: ImageProxy) { - val mediaImage = image.image ?: return + fun analyzeWithoutClosing(image: ImageProxy): Task<*>? { + val mediaImage = image.image ?: return null val inputImage = InputImage.fromMediaImage(mediaImage, image.imageInfo.rotationDegrees) val scanner = BarcodeScanning.getClient() - scanner.process(inputImage) + return scanner.process(inputImage) .addOnSuccessListener { barcodes -> // Throttle callback invocations based on scanThrottleDelay (ms) val now = System.currentTimeMillis() @@ -31,18 +33,21 @@ class QRCodeAnalyzer ( return@addOnSuccessListener } - val strBarcodes = mutableListOf() - barcodes.forEach { barcode -> - strBarcodes.add(barcode ?: return@forEach) - } - - if (strBarcodes.isNotEmpty()) { + if (barcodes.isNotEmpty()) { lastBarcodeDetectedTime = now - onQRCodesDetected(strBarcodes, Size(image.width, image.height)) + onQRCodesDetected(barcodes, Size(image.width, image.height)) } } - .addOnCompleteListener { - image.close() - } + } + + @SuppressLint("UnsafeExperimentalUsageError") + @ExperimentalGetImage + override fun analyze(image: ImageProxy) { + val task = analyzeWithoutClosing(image) + if (task == null) { + image.close() + return + } + task.addOnCompleteListener { image.close() } } } diff --git a/android/src/main/java/com/rncamerakit/events/FaceDetectedEvent.kt b/android/src/main/java/com/rncamerakit/events/FaceDetectedEvent.kt new file mode 100644 index 000000000..4e760254c --- /dev/null +++ b/android/src/main/java/com/rncamerakit/events/FaceDetectedEvent.kt @@ -0,0 +1,36 @@ +package com.rncamerakit.events + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.WritableMap +import com.facebook.react.uimanager.events.Event +import com.rncamerakit.FacePayload + +class FaceDetectedEvent( + surfaceId: Int, + viewId: Int, + private val faces: List, +) : Event(surfaceId, viewId) { + override fun getEventName(): String = EVENT_NAME + + override fun getEventData(): WritableMap { + val array = Arguments.createArray() + for (face in faces) { + val map = Arguments.createMap().apply { + putInt("id", face.id) + putDouble("yaw", face.yaw) + putDouble("pitch", face.pitch) + putDouble("roll", face.roll) + putDouble("boundsX", face.boundsX) + putDouble("boundsY", face.boundsY) + putDouble("boundsWidth", face.boundsWidth) + putDouble("boundsHeight", face.boundsHeight) + } + array.pushMap(map) + } + return Arguments.createMap().apply { putArray("faces", array) } + } + + companion object { + const val EVENT_NAME = "topFaceDetected" + } +} diff --git a/android/src/main/java/com/rncamerakit/events/FaceDetectionInstallStatusEvent.kt b/android/src/main/java/com/rncamerakit/events/FaceDetectionInstallStatusEvent.kt new file mode 100644 index 000000000..25803af3b --- /dev/null +++ b/android/src/main/java/com/rncamerakit/events/FaceDetectionInstallStatusEvent.kt @@ -0,0 +1,21 @@ +package com.rncamerakit.events + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.WritableMap +import com.facebook.react.uimanager.events.Event + +class FaceDetectionInstallStatusEvent( + surfaceId: Int, + viewId: Int, + private val state: String, +) : Event(surfaceId, viewId) { + override fun getEventName(): String = EVENT_NAME + + override fun getEventData(): WritableMap = Arguments.createMap().apply { + putString("state", state) + } + + companion object { + const val EVENT_NAME = "topFaceDetectionInstallStatus" + } +} diff --git a/android/src/newarch/java/com/rncamerakit/CKCameraManager.kt b/android/src/newarch/java/com/rncamerakit/CKCameraManager.kt index cbca7f5e5..d97c3f606 100644 --- a/android/src/newarch/java/com/rncamerakit/CKCameraManager.kt +++ b/android/src/newarch/java/com/rncamerakit/CKCameraManager.kt @@ -8,7 +8,6 @@ import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableType -import com.facebook.react.common.MapBuilder import com.facebook.react.common.ReactConstants.TAG import com.facebook.react.uimanager.SimpleViewManager import com.facebook.react.uimanager.ThemedReactContext @@ -53,14 +52,16 @@ class CKCameraManager(context: ReactApplicationContext) : SimpleViewManager { - return MapBuilder.of( - OrientationChangeEvent.EVENT_NAME, MapBuilder.of("registrationName", "onOrientationChange"), - ReadCodeEvent.EVENT_NAME, MapBuilder.of("registrationName", "onReadCode"), - PictureTakenEvent.EVENT_NAME, MapBuilder.of("registrationName", "onPictureTaken"), - ZoomEvent.EVENT_NAME, MapBuilder.of("registrationName", "onZoom"), - ErrorEvent.EVENT_NAME, MapBuilder.of("registrationName", "onError"), - CaptureButtonPressInEvent.EVENT_NAME, MapBuilder.of("registrationName", "onCaptureButtonPressIn"), - CaptureButtonPressOutEvent.EVENT_NAME, MapBuilder.of("registrationName", "onCaptureButtonPressOut") + return mapOf( + OrientationChangeEvent.EVENT_NAME to mapOf("registrationName" to "onOrientationChange"), + ReadCodeEvent.EVENT_NAME to mapOf("registrationName" to "onReadCode"), + PictureTakenEvent.EVENT_NAME to mapOf("registrationName" to "onPictureTaken"), + ZoomEvent.EVENT_NAME to mapOf("registrationName" to "onZoom"), + ErrorEvent.EVENT_NAME to mapOf("registrationName" to "onError"), + CaptureButtonPressInEvent.EVENT_NAME to mapOf("registrationName" to "onCaptureButtonPressIn"), + CaptureButtonPressOutEvent.EVENT_NAME to mapOf("registrationName" to "onCaptureButtonPressOut"), + FaceDetectedEvent.EVENT_NAME to mapOf("registrationName" to "onFaceDetected"), + FaceDetectionInstallStatusEvent.EVENT_NAME to mapOf("registrationName" to "onFaceDetectionInstallStatus"), ) } @@ -104,6 +105,16 @@ class CKCameraManager(context: ReactApplicationContext) : SimpleViewManager { - return MapBuilder.of( - OrientationChangeEvent.EVENT_NAME, MapBuilder.of("registrationName", "onOrientationChange"), - ReadCodeEvent.EVENT_NAME, MapBuilder.of("registrationName", "onReadCode"), - PictureTakenEvent.EVENT_NAME, MapBuilder.of("registrationName", "onPictureTaken"), - ZoomEvent.EVENT_NAME, MapBuilder.of("registrationName", "onZoom"), - ErrorEvent.EVENT_NAME, MapBuilder.of("registrationName", "onError"), - CaptureButtonPressInEvent.EVENT_NAME, MapBuilder.of("registrationName", "onCaptureButtonPressIn"), - CaptureButtonPressOutEvent.EVENT_NAME, MapBuilder.of("registrationName", "onCaptureButtonPressOut") + return mapOf( + OrientationChangeEvent.EVENT_NAME to mapOf("registrationName" to "onOrientationChange"), + ReadCodeEvent.EVENT_NAME to mapOf("registrationName" to "onReadCode"), + PictureTakenEvent.EVENT_NAME to mapOf("registrationName" to "onPictureTaken"), + ZoomEvent.EVENT_NAME to mapOf("registrationName" to "onZoom"), + ErrorEvent.EVENT_NAME to mapOf("registrationName" to "onError"), + CaptureButtonPressInEvent.EVENT_NAME to mapOf("registrationName" to "onCaptureButtonPressIn"), + CaptureButtonPressOutEvent.EVENT_NAME to mapOf("registrationName" to "onCaptureButtonPressOut"), + FaceDetectedEvent.EVENT_NAME to mapOf("registrationName" to "onFaceDetected"), + FaceDetectionInstallStatusEvent.EVENT_NAME to mapOf("registrationName" to "onFaceDetectionInstallStatus"), ) } @@ -98,6 +99,16 @@ class CKCameraManager(var context: ReactApplicationContext) : SimpleViewManager< view.setScanBarcode(enabled) } + @ReactProp(name = "faceDetectionEnabled") + fun setFaceDetectionEnabled(view: CKCamera, enabled: Boolean) { + view.setFaceDetectionEnabled(enabled) + } + + @ReactProp(name = "faceDetectionThrottleMs") + fun setFaceDetectionThrottleMs(view: CKCamera?, value: Int) { + view?.setFaceDetectionThrottleMs(value) + } + @ReactProp(name = "showFrame") fun setShowFrame(view: CKCamera, enabled: Boolean) { view.setShowFrame(enabled) diff --git a/example/images/faceDetection.png b/example/images/faceDetection.png new file mode 100644 index 000000000..1d9b6e9ac Binary files /dev/null and b/example/images/faceDetection.png differ diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index ba3677e72..d62140197 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -2223,7 +2223,7 @@ PODS: - React-perflogger (= 0.81.0) - React-utils (= 0.81.0) - SocketRocket - - ReactNativeCameraKit (16.2.0): + - ReactNativeCameraKit (17.0.1): - boost - DoubleConversion - fast_float @@ -2552,7 +2552,7 @@ SPEC CHECKSUMS: ReactAppDependencyProvider: c91900fa724baee992f01c05eeb4c9e01a807f78 ReactCodegen: a55799cae416c387aeaae3aabc1bc0289ac19cee ReactCommon: 116d6ee71679243698620d8cd9a9042541e44aa6 - ReactNativeCameraKit: 9a5c627808ea152bc4c7e5574a89ced9ca196e40 + ReactNativeCameraKit: 84894fc476300e5d3b9f4e34f55ff75050ec5050 SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 Yoga: 00013dd9cde63a2d98e8002fcc4f5ddb66c10782 diff --git a/example/src/CameraExample.tsx b/example/src/CameraExample.tsx index cb9cd6aca..5326fd142 100644 --- a/example/src/CameraExample.tsx +++ b/example/src/CameraExample.tsx @@ -1,9 +1,15 @@ import type React from 'react'; -import { useState, useRef, useEffect } from 'react'; -import { StyleSheet, Text, View, TouchableOpacity, Image, Animated, ScrollView } from 'react-native'; +import { useCallback, useState, useRef, useEffect } from 'react'; +import { StyleSheet, Text, View, TouchableOpacity, Image, Animated, ScrollView, type LayoutChangeEvent } from 'react-native'; import Camera from '../../src/Camera'; import { type CameraApi, CameraType, type CaptureData } from '../../src/types'; import { Orientation } from '../../src'; +import { + type FaceData, + type FaceDetectionInstallState, + type OnFaceDetectedData, + type OnFaceDetectionInstallStatusData, +} from '../../src/CameraProps'; import SafeAreaView from './SafeAreaView'; const flashImages = { @@ -33,6 +39,111 @@ function median(values: number[]): number { return sortedValues.length % 2 ? sortedValues[half] : (sortedValues[half - 1] + sortedValues[half]) / 2; } +// "Facing the camera" = head pointed within 15 degrees +// AND the face center is within 20% of the preview center. Bounds are normalized 0–1 with +// top-left origin, so the preview center is (0.5, 0.5) and faceCenter = corner + size/2. +const FACING_THRESHOLD_DEG = 15; +const CENTERING_TOLERANCE = 0.2; +function isFacingCamera(face: FaceData): boolean { + const orientationOk = Math.abs(face.yaw) < FACING_THRESHOLD_DEG && Math.abs(face.pitch) < FACING_THRESHOLD_DEG; + const centerX = face.boundsX + face.boundsWidth / 2; + const centerY = face.boundsY + face.boundsHeight / 2; + const centeredOk = Math.abs(centerX - 0.5) < CENTERING_TOLERANCE && Math.abs(centerY - 0.5) < CENTERING_TOLERANCE; + return orientationOk && centeredOk; +} + +function CaptureButton({ onPress, children }: { onPress: () => void; children?: React.ReactNode }) { + const w = 80; + const brdW = 4; + const spc = 6; + const cInner = 'white'; + const cOuter = 'white'; + return ( + + + + {children} + + ); +} + +function FaceFrame({ + face, + layout, +}: { + face: FaceData; + layout: { x: number; y: number; width: number; height: number }; +}) { + if (!layout.width || !layout.height) return null; + const facing = isFacingCamera(face); + const color = facing ? '#22c55e' : '#facc15'; + const left = layout.x + face.boundsX * layout.width; + const top = layout.y + face.boundsY * layout.height; + const height = face.boundsHeight * layout.height; + return ( + <> + + + ID {face.id} + + + ); +} + +function FaceStats({ faces }: { faces: FaceData[] }) { + const face = faces[0]; + return ( + + Faces: {faces.length} + {face && ( + <> + Yaw: {face.yaw.toFixed(1)}° + Pitch: {face.pitch.toFixed(1)}° + Roll: {face.roll.toFixed(1)}° + Facing: {isFacingCamera(face) ? 'yes' : 'no'} + + Box: {face.boundsX.toFixed(2)},{face.boundsY.toFixed(2)} {face.boundsWidth.toFixed(2)}× + {face.boundsHeight.toFixed(2)} + + + )} + + ); +} + const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolean }) => { const cameraRef = useRef(null); const [currentFlashArrayPosition, setCurrentFlashArrayPosition] = useState(0); @@ -45,6 +156,14 @@ const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolea const [zoom, setZoom] = useState(); const [orientationAnim] = useState(new Animated.Value(3)); const [resize, setResize] = useState<'contain' | 'cover'>('contain'); + const [faceDetection, setFaceDetection] = useState(false); + const [faces, setFaces] = useState([]); + const [cameraLayout, setCameraLayout] = useState({ x: 0, y: 0, width: 0, height: 0 }); + const [faceInstallState, setFaceInstallState] = useState(null); + + useEffect(() => { + if (!faceDetection) setFaces([]); + }, [faceDetection]); // zoom to random positions every 10ms: useEffect(() => { @@ -95,6 +214,27 @@ const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolea setTorchMode(!torchMode); }; + const onSetFaceDetection = () => { + setFaceDetection(!faceDetection); + }; + + const onFaceDetected = useCallback((e: OnFaceDetectedData) => { + const next = e.nativeEvent.faces; + setFaces((prev) => (prev.length === 0 && next.length === 0 ? prev : next)); + }, []); + + const onCameraLayout = useCallback((e: LayoutChangeEvent) => { + const x = e.nativeEvent.layout.x; + const y = e.nativeEvent.layout.y; + const width = e.nativeEvent.layout.width; + const height = e.nativeEvent.layout.height; + setCameraLayout({ x, y, width, height }); + }, []); + + const onFaceDetectionInstallStatus = useCallback((e: OnFaceDetectionInstallStatusData) => { + setFaceInstallState(e.nativeEvent.state); + }, []); + const onCaptureImagePressed = async () => { const times: number[] = []; for (let i = 1; i <= 5; i++) { @@ -123,42 +263,6 @@ const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolea console.log(`median capture time: ${median(times)}ms`); }; - function CaptureButton({ onPress, children }: { onPress: () => void; children?: React.ReactNode }) { - const w = 80; - const brdW = 4; - const spc = 6; - const cInner = 'white'; - const cOuter = 'white'; - return ( - - - - {children} - - ); - } - // Counter-rotate the icons to indicate the actual orientation of the captured photo. // For this example, it'll behave incorrectly since UI orientation is allowed (and already-counter rotates the entire screen) // For real phone apps, lock your UI orientation using a library like 'react-native-orientation-locker' @@ -213,6 +317,16 @@ const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolea /> + + + + void; stress?: boolea torchMode={torchMode ? 'on' : 'off'} shutterPhotoSound maxPhotoQualityPrioritization="speed" + faceDetectionEnabled={faceDetection} + faceDetectionThrottleMs={50} + onLayout={onCameraLayout} + onFaceDetected={onFaceDetected} + onFaceDetectionInstallStatus={onFaceDetectionInstallStatus} onCaptureButtonPressIn={() => { console.log('capture button pressed in'); }} @@ -279,6 +398,29 @@ const CameraExample = ({ onBack, stress }: { onBack: () => void; stress?: boolea }} /> )} + + {faceDetection && !showImageUri && faces.length > 0 && ( + <> + {faces.map((face) => ( + + ))} + + + )} + + {faceDetection && faceInstallState && faceInstallState !== 'ready' && ( + + + {faceInstallState === 'pending' + ? 'Preparing face detection…' + : faceInstallState === 'downloading' + ? 'Downloading face detection…' + : faceInstallState === 'installing' + ? 'Installing face detection…' + : 'Face detection unavailable'} + + + )} @@ -336,10 +478,14 @@ const styles = StyleSheet.create({ justifyContent: 'center', alignItems: 'center', }, + topButtonActive: { + backgroundColor: '#1e7eff', + }, topButtonImg: { margin: 10, width: 24, height: 24, + tintColor: 'white', }, cameraContainer: { justifyContent: 'center', @@ -391,4 +537,50 @@ const styles = StyleSheet.create({ borderRadius: 4, marginEnd: 10, }, + faceFrame: { + position: 'absolute', + borderWidth: 3, + borderRadius: 8, + }, + faceIdBadge: { + position: 'absolute', + paddingHorizontal: 6, + paddingVertical: 2, + borderRadius: 4, + borderWidth: 1, + backgroundColor: 'rgba(0,0,0,0.6)', + }, + faceIdText: { + fontSize: 11, + fontWeight: '600', + fontVariant: ['tabular-nums'], + }, + statsBox: { + position: 'absolute', + top: 10, + right: 25, + backgroundColor: 'rgba(0,0,0,0.55)', + paddingHorizontal: 8, + paddingVertical: 6, + borderRadius: 6, + }, + statsText: { + color: 'white', + fontSize: 11, + fontVariant: ['tabular-nums'], + }, + installBanner: { + position: 'absolute', + top: 30, + left: 20, + right: 20, + backgroundColor: 'rgba(0,0,0,0.7)', + padding: 12, + borderRadius: 6, + }, + installText: { + color: 'white', + fontSize: 13, + textAlign: 'center', + }, }); diff --git a/ios/ReactNativeCameraKit/CKCameraManager.mm b/ios/ReactNativeCameraKit/CKCameraManager.mm index 6ef4da3fc..ef8a78249 100644 --- a/ios/ReactNativeCameraKit/CKCameraManager.mm +++ b/ios/ReactNativeCameraKit/CKCameraManager.mm @@ -33,6 +33,12 @@ @interface RCT_EXTERN_MODULE (CKCameraManager, RCTViewManager) RCT_EXPORT_VIEW_PROPERTY(barcodeFrameSize, NSDictionary) RCT_EXPORT_VIEW_PROPERTY(allowedBarcodeTypes, NSArray) +RCT_EXPORT_VIEW_PROPERTY(faceDetectionEnabled, BOOL) +RCT_EXPORT_VIEW_PROPERTY(faceDetectionThrottleMs, NSInteger) +RCT_EXPORT_VIEW_PROPERTY(onFaceDetected, RCTDirectEventBlock) +// Android-only; Never fired. +RCT_EXPORT_VIEW_PROPERTY(onFaceDetectionInstallStatus, RCTDirectEventBlock) + RCT_EXPORT_VIEW_PROPERTY(onOrientationChange, RCTDirectEventBlock) RCT_EXPORT_VIEW_PROPERTY(onCaptureButtonPressIn, RCTDirectEventBlock) RCT_EXPORT_VIEW_PROPERTY(onCaptureButtonPressOut, RCTDirectEventBlock) diff --git a/ios/ReactNativeCameraKit/CKCameraViewComponentView.mm b/ios/ReactNativeCameraKit/CKCameraViewComponentView.mm index a91fb2fc0..b7fc51133 100644 --- a/ios/ReactNativeCameraKit/CKCameraViewComponentView.mm +++ b/ios/ReactNativeCameraKit/CKCameraViewComponentView.mm @@ -151,6 +151,33 @@ - (void)prepareView { ->onCaptureButtonPressOut({}); } }]; + [_view setOnFaceDetected:^(NSDictionary *event) { + __typeof__(self) strongSelf = weakSelf; + + if (strongSelf != nullptr && strongSelf->_eventEmitter != nullptr) { + NSArray *facesArray = [event valueForKey:@"faces"] ?: @[]; + std::vector faces; + faces.reserve(facesArray.count); + for (NSDictionary *face in facesArray) { + faces.push_back({ + .id = [face[@"id"] intValue], + .yaw = [face[@"yaw"] doubleValue], + .pitch = [face[@"pitch"] doubleValue], + .roll = [face[@"roll"] doubleValue], + .boundsX = [face[@"boundsX"] doubleValue], + .boundsY = [face[@"boundsY"] doubleValue], + .boundsWidth = [face[@"boundsWidth"] doubleValue], + .boundsHeight = [face[@"boundsHeight"] doubleValue], + }); + } + facebook::react::CKCameraEventEmitter::OnFaceDetected payload = { + .faces = std::move(faces), + }; + std::dynamic_pointer_cast( + strongSelf->_eventEmitter) + ->onFaceDetected(payload); + } + }]; self.contentView = _view; } @@ -291,6 +318,15 @@ - (void)updateProps:(const Props::Shared &)props @{@"width" : @(barcodeWidth), @"height" : @(barcodeHeight)}; [changedProps addObject:@"barcodeFrameSize"]; } + if (_view.faceDetectionEnabled != newProps.faceDetectionEnabled) { + _view.faceDetectionEnabled = newProps.faceDetectionEnabled; + [changedProps addObject:@"faceDetectionEnabled"]; + } + if (newProps.faceDetectionThrottleMs > -1 && + _view.faceDetectionThrottleMs != newProps.faceDetectionThrottleMs) { + _view.faceDetectionThrottleMs = newProps.faceDetectionThrottleMs; + [changedProps addObject:@"faceDetectionThrottleMs"]; + } // Since viewprops optional props isn't supported in all RN versions, // we assume empty arrays mean it's not defined / ignore changes to it. // if the user/dev wants to NOT define the prop, they can simply use diff --git a/ios/ReactNativeCameraKit/CameraProtocol.swift b/ios/ReactNativeCameraKit/CameraProtocol.swift index a0c2353c2..1cf943a42 100644 --- a/ios/ReactNativeCameraKit/CameraProtocol.swift +++ b/ios/ReactNativeCameraKit/CameraProtocol.swift @@ -34,6 +34,12 @@ protocol CameraProtocol: AnyObject, FocusInterfaceViewDelegate { func update(scannerFrameSize: CGRect?) + func isFaceDetectionEnabled( + _ isEnabled: Bool, + onFaceDetected: ((_ payloads: [FaceDetectionPayload]) -> Void)?) + + func update(faceDetectionThrottleMs: Int) + func capturePicture( onWillCapture: @escaping () -> Void, onSuccess: diff --git a/ios/ReactNativeCameraKit/CameraView.swift b/ios/ReactNativeCameraKit/CameraView.swift index 1e5cc62d2..28720e734 100644 --- a/ios/ReactNativeCameraKit/CameraView.swift +++ b/ios/ReactNativeCameraKit/CameraView.swift @@ -49,6 +49,13 @@ public class CameraView: UIView { @objc public var barcodeFrameSize: NSDictionary? @objc public var allowedBarcodeTypes: NSArray? + // face detection + @objc public var faceDetectionEnabled = false + @objc public var faceDetectionThrottleMs: Int = FaceDetector.defaultThrottleMs + @objc public var onFaceDetected: RCTDirectEventBlock? + // Android-only; Never fired. + @objc public var onFaceDetectionInstallStatus: RCTDirectEventBlock? + // other @objc public var onOrientationChange: RCTDirectEventBlock? @objc public var onZoom: RCTDirectEventBlock? @@ -275,6 +282,18 @@ public class CameraView: UIView { }) } + // Face detection + if changedProps.contains("faceDetectionEnabled") || changedProps.contains("onFaceDetected") { + camera.isFaceDetectionEnabled( + faceDetectionEnabled, + onFaceDetected: onFaceDetected == nil ? nil : { [weak self] payloads in + self?.onFaceDetected?(["faces": payloads.map { $0.asDictionary }]) + }) + } + if changedProps.contains("faceDetectionThrottleMs") { + camera.update(faceDetectionThrottleMs: faceDetectionThrottleMs) + } + if changedProps.contains("showFrame") || changedProps.contains("scanBarcode") { DispatchQueue.main.async { self.scannerInterfaceView.isHidden = !self.showFrame diff --git a/ios/ReactNativeCameraKit/FaceDetector.swift b/ios/ReactNativeCameraKit/FaceDetector.swift new file mode 100644 index 000000000..3b16a507f --- /dev/null +++ b/ios/ReactNativeCameraKit/FaceDetector.swift @@ -0,0 +1,103 @@ +// +// FaceDetector.swift +// ReactNativeCameraKit +// + +import AVFoundation +import CoreVideo +import Foundation +import Vision + +struct FaceDetectionPayload { + let id: Int + let yaw: Double + let pitch: Double + let roll: Double + let boundsX: Double + let boundsY: Double + let boundsWidth: Double + let boundsHeight: Double + + var asDictionary: [String: Any] { + return [ + "id": id, + "yaw": yaw, + "pitch": pitch, + "roll": roll, + "boundsX": boundsX, + "boundsY": boundsY, + "boundsWidth": boundsWidth, + "boundsHeight": boundsHeight, + ] + } +} + +final class FaceDetector { + static let defaultThrottleMs: Int = 100 + + private var throttleSeconds: TimeInterval = TimeInterval(FaceDetector.defaultThrottleMs) / 1000.0 + + private let request: VNDetectFaceRectanglesRequest = { + let r = VNDetectFaceRectanglesRequest() + r.revision = VNDetectFaceRectanglesRequestRevision3 + return r + }() + + private var lastEmit: TimeInterval = 0 + private var lastErrorDescription: String? + + func update(throttleMs: Int) { + let validated = throttleMs < 0 ? FaceDetector.defaultThrottleMs : throttleMs + throttleSeconds = TimeInterval(validated) / 1000.0 + } + + func process(pixelBuffer: CVPixelBuffer, orientation: CGImagePropertyOrientation) -> [FaceDetectionPayload]? { + let now = Date.timeIntervalSinceReferenceDate + guard now - lastEmit >= throttleSeconds else { return nil } + lastEmit = now + + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: orientation, options: [:]) + do { + try handler.perform([request]) + lastErrorDescription = nil + } catch { + let description = "\(error)" + if description != lastErrorDescription { + print("CKCameraKit: face detection error: \(description)") + lastErrorDescription = description + } + return [] + } + + let observations = request.results ?? [] + return observations.enumerated().map { build(from: $1, id: $0) } + } + + private func build(from face: VNFaceObservation, id: Int) -> FaceDetectionPayload { + let bounds = previewBounds(face.boundingBox) + return FaceDetectionPayload( + id: id, + yaw: -degrees(from: face.yaw), + pitch: degrees(from: face.pitch), + roll: degrees(from: face.roll), + boundsX: Double(bounds.origin.x), + boundsY: Double(bounds.origin.y), + boundsWidth: Double(bounds.size.width), + boundsHeight: Double(bounds.size.height) + ) + } + + private func degrees(from radians: NSNumber?) -> Double { + guard let radians = radians?.doubleValue else { return 0 } + return radians * 180.0 / .pi + } + + private func previewBounds(_ rect: CGRect) -> CGRect { + return CGRect( + x: rect.origin.x, + y: 1.0 - rect.origin.y - rect.size.height, + width: rect.size.width, + height: rect.size.height + ) + } +} diff --git a/ios/ReactNativeCameraKit/RealCamera.swift b/ios/ReactNativeCameraKit/RealCamera.swift index 135a383ce..cc722c67d 100644 --- a/ios/ReactNativeCameraKit/RealCamera.swift +++ b/ios/ReactNativeCameraKit/RealCamera.swift @@ -7,6 +7,7 @@ import AVFoundation import CoreMotion +import ImageIO import React import UIKit @@ -14,13 +15,14 @@ import UIKit * Real camera implementation that uses AVFoundation */ // swiftlint:disable:next type_body_length -class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelegate { +class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelegate, AVCaptureVideoDataOutputSampleBufferDelegate { var previewView: UIView { cameraPreview } private let cameraPreview = RealPreviewView(frame: .zero) private let session = AVCaptureSession() // Communicate with the session and other session objects on this queue. private let sessionQueue = DispatchQueue(label: "com.tesla.react-native-camera-kit") + private let visionQueue = DispatchQueue(label: "com.tesla.react-native-camera-kit.vision") // utilities private var setupResult: SetupResult = .notStarted @@ -30,6 +32,11 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega private var videoDeviceInput: AVCaptureDeviceInput? private let photoOutput = AVCapturePhotoOutput() private let metadataOutput = AVCaptureMetadataOutput() + private let videoDataOutput = AVCaptureVideoDataOutput() + private var isVideoDataOutputAttached = false + private let faceDetector = FaceDetector() + private var onFaceDetected: (([FaceDetectionPayload]) -> Void)? + private var currentVisionOrientation: CGImagePropertyOrientation = .right private var resizeMode: ResizeMode = .contain private var flashMode: FlashMode = .auto @@ -343,6 +350,7 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega } self.addObservers() + self.refreshVisionOrientation() } } @@ -467,6 +475,43 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega } } + func update(faceDetectionThrottleMs: Int) { + visionQueue.async { + self.faceDetector.update(throttleMs: faceDetectionThrottleMs) + } + } + + func isFaceDetectionEnabled( + _ isEnabled: Bool, + onFaceDetected: ((_ payloads: [FaceDetectionPayload]) -> Void)? + ) { + sessionQueue.async { + self.onFaceDetected = onFaceDetected + + let shouldAttach = isEnabled && onFaceDetected != nil + if shouldAttach == self.isVideoDataOutputAttached { return } + + self.session.beginConfiguration() + defer { self.session.commitConfiguration() } + + if shouldAttach { + self.videoDataOutput.alwaysDiscardsLateVideoFrames = true + self.videoDataOutput.videoSettings = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + ] + self.videoDataOutput.setSampleBufferDelegate(self, queue: self.visionQueue) + if self.session.canAddOutput(self.videoDataOutput) { + self.session.addOutput(self.videoDataOutput) + self.isVideoDataOutputAttached = true + } + } else { + self.videoDataOutput.setSampleBufferDelegate(nil, queue: nil) + self.session.removeOutput(self.videoDataOutput) + self.isVideoDataOutputAttached = false + } + } + } + // MARK: - AVCaptureMetadataOutputObjectsDelegate func metadataOutput( @@ -487,6 +532,67 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega onBarcodeRead?(codeStringValue, barcodeType) } + // MARK: - Vision orientation + + private func visionOrientation(for cameraPosition: AVCaptureDevice.Position) -> CGImagePropertyOrientation { + let isFront = cameraPosition == .front + switch deviceOrientation { + case .landscapeLeft: return isFront ? .downMirrored : .up + case .landscapeRight: return isFront ? .upMirrored : .down + case .portraitUpsideDown: return isFront ? .rightMirrored : .left + default: return isFront ? .leftMirrored : .right + } + } + + private func refreshVisionOrientation() { + guard let position = videoDeviceInput?.device.position else { return } + let newOrientation = visionOrientation(for: position) + visionQueue.async { + self.currentVisionOrientation = newOrientation + } + } + + // MARK: - AVCaptureVideoDataOutputSampleBufferDelegate + + func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + + guard let payloads = faceDetector.process( + pixelBuffer: pixelBuffer, + orientation: currentVisionOrientation + ) else { + return + } + + DispatchQueue.main.async { + let layer = self.cameraPreview.previewLayer + let layerSize = layer.bounds.size + guard layerSize.width > 0, layerSize.height > 0 else { return } + let frameRect = layer.layerRectConverted( + fromMetadataOutputRect: CGRect(x: 0, y: 0, width: 1, height: 1)) + guard frameRect.width > 0, frameRect.height > 0 else { return } + + let converted = payloads.map { p -> FaceDetectionPayload in + let layerX = frameRect.origin.x + p.boundsX * frameRect.width + let layerY = frameRect.origin.y + p.boundsY * frameRect.height + let layerW = p.boundsWidth * frameRect.width + let layerH = p.boundsHeight * frameRect.height + return FaceDetectionPayload( + id: p.id, yaw: p.yaw, pitch: p.pitch, roll: p.roll, + boundsX: Double(layerX / layerSize.width), + boundsY: Double(layerY / layerSize.height), + boundsWidth: Double(layerW / layerSize.width), + boundsHeight: Double(layerH / layerSize.height) + ) + } + self.onFaceDetected?(converted) + } + } + // MARK: - Private private func videoOrientation(from deviceOrientation: UIDeviceOrientation) @@ -579,6 +685,7 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega if session.canAddInput(videoDeviceInput) { session.addInput(videoDeviceInput) self.videoDeviceInput = videoDeviceInput + self.refreshVisionOrientation() } else { return .sessionConfigurationFailed } @@ -719,6 +826,7 @@ class RealCamera: NSObject, CameraProtocol, AVCaptureMetadataOutputObjectsDelega } self.deviceOrientation = newOrientation + self.refreshVisionOrientation() self.onOrientationChange?([ "orientation": Orientation.init(from: newOrientation)!.rawValue ]) diff --git a/ios/ReactNativeCameraKit/SimulatorCamera.swift b/ios/ReactNativeCameraKit/SimulatorCamera.swift index d72a06017..41fe9629e 100644 --- a/ios/ReactNativeCameraKit/SimulatorCamera.swift +++ b/ios/ReactNativeCameraKit/SimulatorCamera.swift @@ -191,6 +191,13 @@ class SimulatorCamera: CameraProtocol { ) {} func update(scannerFrameSize: CGRect?) {} + func isFaceDetectionEnabled( + _ isEnabled: Bool, + onFaceDetected: ((_ payloads: [FaceDetectionPayload]) -> Void)? + ) {} + + func update(faceDetectionThrottleMs: Int) {} + func capturePicture( onWillCapture: @escaping () -> Void, onSuccess: diff --git a/src/Camera.android.tsx b/src/Camera.android.tsx index 56cddbccd..0a47aff6d 100644 --- a/src/Camera.android.tsx +++ b/src/Camera.android.tsx @@ -14,6 +14,7 @@ const Camera = React.forwardRef((props, ref) => { props.zoom = props.zoom ?? -1; props.maxZoom = props.maxZoom ?? -1; props.scanThrottleDelay = props.scanThrottleDelay ?? -1; + props.faceDetectionThrottleMs = props.faceDetectionThrottleMs ?? -1; props.allowedBarcodeTypes = props.allowedBarcodeTypes ?? supportedCodeFormats; diff --git a/src/Camera.ios.tsx b/src/Camera.ios.tsx index 0845d6d91..63d3c87b6 100644 --- a/src/Camera.ios.tsx +++ b/src/Camera.ios.tsx @@ -14,6 +14,7 @@ const Camera = React.forwardRef((props, ref) => { props.zoom = props.zoom ?? -1; props.maxZoom = props.maxZoom ?? -1; props.scanThrottleDelay = props.scanThrottleDelay ?? -1; + props.faceDetectionThrottleMs = props.faceDetectionThrottleMs ?? -1; props.iOsDeferredStart = props.iOsDeferredStart ?? true; props.allowedBarcodeTypes = props.allowedBarcodeTypes ?? supportedCodeFormats; diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 2eec24af1..aa33dc254 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -27,7 +27,38 @@ export type OnZoom = { nativeEvent: { zoom: number; }; -} +}; + +export type FaceData = { + id: number; + yaw: number; + pitch: number; + roll: number; + boundsX: number; + boundsY: number; + boundsWidth: number; + boundsHeight: number; +}; + +export type OnFaceDetectedData = { + nativeEvent: { + faces: FaceData[]; + }; +}; + +export type FaceDetectionInstallState = + | 'pending' + | 'downloading' + | 'installing' + | 'ready' + | 'failed' + | 'unavailable'; + +export type OnFaceDetectionInstallStatusData = { + nativeEvent: { + state: FaceDetectionInstallState; + }; +}; export interface CameraProps extends ViewProps { // Behavior @@ -120,4 +151,12 @@ export interface CameraProps extends ViewProps { onCaptureButtonPressIn?: ({ nativeEvent: {} }) => void; onCaptureButtonPressOut?: ({ nativeEvent: {} }) => void; allowedBarcodeTypes?: CodeFormat[]; + /** Enable real-time face detection. iOS uses Apple Vision; Android uses MLKit */ + faceDetectionEnabled?: boolean; + /** Throttle how often `onFaceDetected` emits. Defaults to 100 (~10 events/sec) */ + faceDetectionThrottleMs?: number; + /** Fires per frame while face detection is active; bounds are normalized 0–1 in preview space */ + onFaceDetected?: (event: OnFaceDetectedData) => void; + /** **Android only**. Fires while the MLKit face detection module is being downloaded by Play Services on first use */ + onFaceDetectionInstallStatus?: (event: OnFaceDetectionInstallStatusData) => void; } diff --git a/src/specs/CameraNativeComponent.ts b/src/specs/CameraNativeComponent.ts index e9d5d7dc1..ab5a71522 100644 --- a/src/specs/CameraNativeComponent.ts +++ b/src/specs/CameraNativeComponent.ts @@ -28,6 +28,19 @@ type OnZoom = { zoom: Double; } +type OnFaceDetectedData = { + faces: { + id: Int32; + yaw: Double; + pitch: Double; + roll: Double; + boundsX: Double; + boundsY: Double; + boundsWidth: Double; + boundsHeight: Double; + }[]; +}; + // We have to use -1 until RN Fabric (New Arch for view components) supports optional values: // https://github.com/facebook/react-native/issues/49920#issuecomment-3237917813 export interface NativeProps extends ViewProps { @@ -59,6 +72,10 @@ export interface NativeProps extends ViewProps { onCaptureButtonPressIn?: DirectEventHandler<{}>; onCaptureButtonPressOut?: DirectEventHandler<{}>; allowedBarcodeTypes?: string[]; + faceDetectionEnabled?: boolean; + faceDetectionThrottleMs?: WithDefault; + onFaceDetected?: DirectEventHandler; + onFaceDetectionInstallStatus?: DirectEventHandler<{ state: string }>; // not mentioned in props but available on the native side shutterAnimationDuration?: WithDefault;