/** * useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe * FaceLandmarker WASM task and exposes a frame-processing callback. * * The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default * CDN path). Initialization is async; `isReady` is false until complete. * * Usage: * const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker(); * // In rAF loop: * const result = detectFrame(videoElement, performance.now()); */ import { useState, useEffect, useRef, useCallback } from 'react'; // MediaPipe tasks-vision is loaded dynamically to avoid SSR issues. // Types are inlined below so we don't need @types/mediapipe type FaceLandmarker = { detectForVideo: ( videoElement: HTMLVideoElement, timestamp: number, ) => FaceLandmarkerResult; close: () => void; }; export interface BlendShapeCategory { categoryName: string; score: number; displayName: string; index: number; } export interface FaceLandmarkerResult { faceBlendshapes: Array<{ categories: BlendShapeCategory[] }>; faceLandmarks: Array>; } const MODEL_URL = import.meta.env.VITE_MEDIAPIPE_MODEL_URL ?? 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task'; interface UseFaceLandmarkerReturn { isLoading: boolean; isReady: boolean; error: string | null; detectFrame: ( videoElement: HTMLVideoElement, timestampMs: number, ) => FaceLandmarkerResult | null; } export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn { const landmarkerRef = useRef(null); const [isLoading, setIsLoading] = useState(true); const [isReady, setIsReady] = useState(false); const [error, setError] = useState(null); useEffect(() => { let cancelled = false; async function init() { try { // Dynamic import — avoids pulling the WASM runtime into the main bundle const vision = await import( /* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs' ); const { FaceLandmarker, FilesetResolver } = vision as { FaceLandmarker: { createFromOptions: ( resolver: unknown, options: unknown, ) => Promise; }; FilesetResolver: { forVisionTasks: (path: string) => Promise }; }; const filesetResolver = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm', ); const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, { baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU', }, outputFaceBlendshapes: true, runningMode: 'VIDEO', numFaces: 1, }); if (!cancelled) { landmarkerRef.current = landmarker; setIsLoading(false); setIsReady(true); } else { landmarker.close(); } } catch (err) { if (!cancelled) { console.error('[MediaPipe] Initialization failed:', err); setError( err instanceof Error ? err.message : 'MediaPipe failed to initialize.', ); setIsLoading(false); } } } void init(); return () => { cancelled = true; landmarkerRef.current?.close(); landmarkerRef.current = null; }; }, []); const detectFrame = useCallback( (videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => { if (!landmarkerRef.current || !isReady) return null; try { return landmarkerRef.current.detectForVideo(videoElement, timestampMs); } catch { return null; } }, [isReady], ); return { isLoading, isReady, error, detectFrame }; }