Built the Sentinel Tab

app/src/hooks/useMediapipeFaceLandmarker.ts

@@ -0,0 +1,133 @@
+/**
+ * useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
+ * FaceLandmarker WASM task and exposes a frame-processing callback.
+ *
+ * The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
+ * CDN path). Initialization is async; `isReady` is false until complete.
+ *
+ * Usage:
+ *   const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
+ *   // In rAF loop:
+ *   const result = detectFrame(videoElement, performance.now());
+ */
+
+import { useState, useEffect, useRef, useCallback } from 'react';
+
+// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
+// Types are inlined below so we don't need @types/mediapipe
+type FaceLandmarker = {
+  detectForVideo: (
+    videoElement: HTMLVideoElement,
+    timestamp: number,
+  ) => FaceLandmarkerResult;
+  close: () => void;
+};
+
+export interface BlendShapeCategory {
+  categoryName: string;
+  score: number;
+  displayName: string;
+  index: number;
+}
+
+export interface FaceLandmarkerResult {
+  faceBlendshapes: Array<{ categories: BlendShapeCategory[] }>;
+  faceLandmarks: Array<Array<{ x: number; y: number; z: number }>>;
+}
+
+const MODEL_URL =
+  import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
+  'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
+
+interface UseFaceLandmarkerReturn {
+  isLoading: boolean;
+  isReady: boolean;
+  error: string | null;
+  detectFrame: (
+    videoElement: HTMLVideoElement,
+    timestampMs: number,
+  ) => FaceLandmarkerResult | null;
+}
+
+export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
+  const landmarkerRef = useRef<FaceLandmarker | null>(null);
+  const [isLoading, setIsLoading] = useState(true);
+  const [isReady, setIsReady] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+
+    async function init() {
+      try {
+        // Dynamic import — avoids pulling the WASM runtime into the main bundle
+        const vision = await import(
+          /* @vite-ignore */
+          'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
+        );
+
+        const { FaceLandmarker, FilesetResolver } = vision as {
+          FaceLandmarker: {
+            createFromOptions: (
+              resolver: unknown,
+              options: unknown,
+            ) => Promise<FaceLandmarker>;
+          };
+          FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
+        };
+
+        const filesetResolver = await FilesetResolver.forVisionTasks(
+          'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
+        );
+
+        const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
+          baseOptions: {
+            modelAssetPath: MODEL_URL,
+            delegate: 'GPU',
+          },
+          outputFaceBlendshapes: true,
+          runningMode: 'VIDEO',
+          numFaces: 1,
+        });
+
+        if (!cancelled) {
+          landmarkerRef.current = landmarker;
+          setIsLoading(false);
+          setIsReady(true);
+        } else {
+          landmarker.close();
+        }
+      } catch (err) {
+        if (!cancelled) {
+          console.error('[MediaPipe] Initialization failed:', err);
+          setError(
+            err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
+          );
+          setIsLoading(false);
+        }
+      }
+    }
+
+    void init();
+
+    return () => {
+      cancelled = true;
+      landmarkerRef.current?.close();
+      landmarkerRef.current = null;
+    };
+  }, []);
+
+  const detectFrame = useCallback(
+    (videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
+      if (!landmarkerRef.current || !isReady) return null;
+      try {
+        return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
+      } catch {
+        return null;
+      }
+    },
+    [isReady],
+  );
+
+  return { isLoading, isReady, error, detectFrame };
+}