134 lines
3.9 KiB
TypeScript
134 lines
3.9 KiB
TypeScript
/**
|
|
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
|
|
* FaceLandmarker WASM task and exposes a frame-processing callback.
|
|
*
|
|
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
|
|
* CDN path). Initialization is async; `isReady` is false until complete.
|
|
*
|
|
* Usage:
|
|
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
|
|
* // In rAF loop:
|
|
* const result = detectFrame(videoElement, performance.now());
|
|
*/
|
|
|
|
import { useState, useEffect, useRef, useCallback } from 'react';
|
|
|
|
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
|
|
// Types are inlined below so we don't need @types/mediapipe
|
|
type FaceLandmarker = {
|
|
detectForVideo: (
|
|
videoElement: HTMLVideoElement,
|
|
timestamp: number,
|
|
) => FaceLandmarkerResult;
|
|
close: () => void;
|
|
};
|
|
|
|
export interface BlendShapeCategory {
|
|
categoryName: string;
|
|
score: number;
|
|
displayName: string;
|
|
index: number;
|
|
}
|
|
|
|
export interface FaceLandmarkerResult {
|
|
faceBlendshapes: Array<{ categories: BlendShapeCategory[] }>;
|
|
faceLandmarks: Array<Array<{ x: number; y: number; z: number }>>;
|
|
}
|
|
|
|
const MODEL_URL =
|
|
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
|
|
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
|
|
|
|
interface UseFaceLandmarkerReturn {
|
|
isLoading: boolean;
|
|
isReady: boolean;
|
|
error: string | null;
|
|
detectFrame: (
|
|
videoElement: HTMLVideoElement,
|
|
timestampMs: number,
|
|
) => FaceLandmarkerResult | null;
|
|
}
|
|
|
|
export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
|
const landmarkerRef = useRef<FaceLandmarker | null>(null);
|
|
const [isLoading, setIsLoading] = useState(true);
|
|
const [isReady, setIsReady] = useState(false);
|
|
const [error, setError] = useState<string | null>(null);
|
|
|
|
useEffect(() => {
|
|
let cancelled = false;
|
|
|
|
async function init() {
|
|
try {
|
|
// Dynamic import — avoids pulling the WASM runtime into the main bundle
|
|
const vision = await import(
|
|
/* @vite-ignore */
|
|
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
|
|
);
|
|
|
|
const { FaceLandmarker, FilesetResolver } = vision as {
|
|
FaceLandmarker: {
|
|
createFromOptions: (
|
|
resolver: unknown,
|
|
options: unknown,
|
|
) => Promise<FaceLandmarker>;
|
|
};
|
|
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
|
|
};
|
|
|
|
const filesetResolver = await FilesetResolver.forVisionTasks(
|
|
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
|
|
);
|
|
|
|
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
|
baseOptions: {
|
|
modelAssetPath: MODEL_URL,
|
|
delegate: 'GPU',
|
|
},
|
|
outputFaceBlendshapes: true,
|
|
runningMode: 'VIDEO',
|
|
numFaces: 1,
|
|
});
|
|
|
|
if (!cancelled) {
|
|
landmarkerRef.current = landmarker;
|
|
setIsLoading(false);
|
|
setIsReady(true);
|
|
} else {
|
|
landmarker.close();
|
|
}
|
|
} catch (err) {
|
|
if (!cancelled) {
|
|
console.error('[MediaPipe] Initialization failed:', err);
|
|
setError(
|
|
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
|
|
);
|
|
setIsLoading(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
void init();
|
|
|
|
return () => {
|
|
cancelled = true;
|
|
landmarkerRef.current?.close();
|
|
landmarkerRef.current = null;
|
|
};
|
|
}, []);
|
|
|
|
const detectFrame = useCallback(
|
|
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
|
|
if (!landmarkerRef.current || !isReady) return null;
|
|
try {
|
|
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
|
|
} catch {
|
|
return null;
|
|
}
|
|
},
|
|
[isReady],
|
|
);
|
|
|
|
return { isLoading, isReady, error, detectFrame };
|
|
}
|