From 3c0a7bceeff36d3995b6f4fa83a9bdee19033f39 Mon Sep 17 00:00:00 2001
From: Kai Niebes <kai.niebes@outlook.com>
Date: Thu, 11 Jun 2026 19:58:48 +0200
Subject: [PATCH] Add screen-space picking for point cloud annotations

---
 .../services/annotation/annotation.service.ts |  9 +++
 .../helpers/find-closest-cloud-point.ts       | 60 +++++++++++++++++++
 .../babylon/importers/copc/copc-importer.ts   |  5 +-
 3 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 src/app/services/annotation/helpers/find-closest-cloud-point.ts

diff --git a/src/app/services/annotation/annotation.service.ts b/src/app/services/annotation/annotation.service.ts
index f0df5ea8..82b3a5a5 100644
--- a/src/app/services/annotation/annotation.service.ts
+++ b/src/app/services/annotation/annotation.service.ts
@@ -34,6 +34,7 @@ import { UserdataService } from '../userdata/userdata.service';
 import { createMarker, createMarkerName } from './visual3DElements';
 import { ReorderMovement } from 'src/app/components/entity-feature-annotations/annotation/annotation.component';
 import { ExtenderTransformer } from '@kompakkt/plugins/extender';
+import { findClosestCloudPoint } from './helpers/find-closest-cloud-point';
 import { findClosestSplatPoint } from './helpers/find-closest-splat';
 import ObjectID from 'bson-objectid';
 import {
@@ -368,6 +369,14 @@ export class AnnotationService {
             const normalVector = closestSplat.subtract(camera.position).normalize();
             return { pickedPoint: closestSplat, pickedNormal: normalVector };
           }
+          case 'cloud': {
+            const closestPoint = findClosestCloudPoint(scene);
+            if (!closestPoint) return null;
+            const camera = this.babylon.getActiveCamera();
+            if (!camera) return;
+            const normalVector = closestPoint.subtract(camera.position).normalize();
+            return { pickedPoint: closestPoint, pickedNormal: normalVector };
+          }
           default: {
             const result = scene.pick(scene.pointerX, scene.pointerY, mesh => mesh.isPickable);
             if (!result?.pickedPoint) return;
diff --git a/src/app/services/annotation/helpers/find-closest-cloud-point.ts b/src/app/services/annotation/helpers/find-closest-cloud-point.ts
new file mode 100644
index 00000000..6a67e35e
--- /dev/null
+++ b/src/app/services/annotation/helpers/find-closest-cloud-point.ts
@@ -0,0 +1,60 @@
+import { Matrix, Mesh, Nullable, Scene, Vector3, VertexBuffer, Viewport } from '@babylonjs/core';
+
+const MAX_PICKING_DISTANCE_SCREEN_SQ = 30 * 30;
+
+export const findClosestCloudPoint = (scene: Scene): Nullable<Vector3> => {
+  const { pointerX, pointerY } = scene;
+  const camera = scene.activeCamera;
+  if (!camera) return null;
+
+  const engine = scene.getEngine();
+  const viewport = new Viewport(0, 0, engine.getRenderWidth(), engine.getRenderHeight());
+  const transformMatrix = scene.getTransformMatrix();
+
+  // Walk every per-octree-cell point mesh (one Mesh per COPC/EPT octree node,
+  // named 'point-cloud-<level>-<x>-<y>-<z>'). The parent 'root' TransformNode
+  // is excluded by the name filter; the resolved root 'point-cloud-0-0-0-0'
+  // is included.
+  const pointMeshes = scene.meshes.filter(
+    (mesh): mesh is Mesh => mesh instanceof Mesh && mesh.name.startsWith('point-cloud-'),
+  );
+  if (pointMeshes.length === 0) return null;
+
+  let closestPoint: Nullable<Vector3> = null;
+  let minScreenDistanceSq = Infinity;
+  let closestDepth = Infinity;
+
+  for (const mesh of pointMeshes) {
+    const positions = mesh.getVerticesData(VertexBuffer.PositionKind);
+    if (!positions || positions.length === 0) continue;
+
+    const worldMatrix = mesh.getWorldMatrix();
+
+    for (let i = 0; i < positions.length; i += 3) {
+      const local = new Vector3(positions[i], positions[i + 1], positions[i + 2]);
+      const world = Vector3.TransformCoordinates(local, worldMatrix);
+
+      // Note: the first matrix arg is the world matrix of the object whose
+      // local-space vertices are being projected. The point is already in
+      // world space, so we pass IdentityReadOnly.
+      const screen = Vector3.Project(world, Matrix.IdentityReadOnly, transformMatrix, viewport);
+
+      const dx = screen.x - pointerX;
+      const dy = screen.y - pointerY;
+      const distSq = dx * dx + dy * dy;
+
+      if (distSq < MAX_PICKING_DISTANCE_SCREEN_SQ) {
+        if (
+          distSq < minScreenDistanceSq ||
+          (Math.abs(distSq - minScreenDistanceSq) < 1e-5 && screen.z < closestDepth)
+        ) {
+          minScreenDistanceSq = distSq;
+          closestPoint = world;
+          closestDepth = screen.z;
+        }
+      }
+    }
+  }
+
+  return closestPoint;
+};
diff --git a/src/app/services/babylon/importers/copc/copc-importer.ts b/src/app/services/babylon/importers/copc/copc-importer.ts
index f96e8b69..58c2e564 100644
--- a/src/app/services/babylon/importers/copc/copc-importer.ts
+++ b/src/app/services/babylon/importers/copc/copc-importer.ts
@@ -79,10 +79,11 @@ export class CopcImporter implements ISceneLoaderPluginAsync {
       return url.startsWith('/') ? new URL(url, window.location.origin).toString() : url;
     })();
     const copc = await Copc.create(filename);
+    console.log('COPC:', copc);
     const { nodes, pages } = await retryWithBackoff(() =>
       Copc.loadHierarchyPage(filename, copc.info.rootHierarchyPage),
     );
-    console.log(copc, nodes, pages);
+    console.log('COPC nodes & pages:', nodes, pages);
 
     PointCloudImporter.maxLOD = Math.max(...Object.keys(nodes).map(key => +key[0]));
     PointCloudImporter.totalPoints = Object.values(nodes).reduce(
@@ -283,7 +284,7 @@ export class CopcImporter implements ISceneLoaderPluginAsync {
       return mesh;
     });
     console.log('rootMesh', rootMesh);
-    loadNextLevelOfDetail();
+    void loadNextLevelOfDetail();
 
     // Recalculate bounding box of rootNodeMesh
     rootNodeMesh.refreshBoundingInfo();