fix/#22 (#23)
Co-authored-by: Sagnik <sagnik7896@gmail.com> Reviewed-on: #23
This commit was merged in pull request #23.
This commit is contained in:
@@ -1,27 +1,8 @@
|
||||
/**
|
||||
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
|
||||
* FaceLandmarker WASM task and exposes a frame-processing callback.
|
||||
*
|
||||
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
|
||||
* CDN path). Initialization is async; `isReady` is false until complete.
|
||||
*
|
||||
* Usage:
|
||||
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
|
||||
* // In rAF loop:
|
||||
* const result = detectFrame(videoElement, performance.now());
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useRef, useCallback } from 'react';
|
||||
|
||||
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
|
||||
// Types are inlined below so we don't need @types/mediapipe
|
||||
type FaceLandmarker = {
|
||||
detectForVideo: (
|
||||
videoElement: HTMLVideoElement,
|
||||
timestamp: number,
|
||||
) => FaceLandmarkerResult;
|
||||
close: () => void;
|
||||
};
|
||||
import {
|
||||
FaceLandmarker,
|
||||
FilesetResolver,
|
||||
} from '@mediapipe/tasks-vision';
|
||||
|
||||
export interface BlendShapeCategory {
|
||||
categoryName: string;
|
||||
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
|
||||
|
||||
const MODEL_URL =
|
||||
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
|
||||
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
|
||||
'/mediapipe/assets/face_landmarker.task';
|
||||
|
||||
const WASM_ROOT =
|
||||
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
|
||||
'/mediapipe/wasm';
|
||||
|
||||
interface UseFaceLandmarkerReturn {
|
||||
isLoading: boolean;
|
||||
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
// Dynamic import — avoids pulling the WASM runtime into the main bundle
|
||||
const vision = await import(
|
||||
/* @vite-ignore */
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
|
||||
);
|
||||
|
||||
const { FaceLandmarker, FilesetResolver } = vision as {
|
||||
FaceLandmarker: {
|
||||
createFromOptions: (
|
||||
resolver: unknown,
|
||||
options: unknown,
|
||||
) => Promise<FaceLandmarker>;
|
||||
};
|
||||
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
|
||||
};
|
||||
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
|
||||
);
|
||||
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
|
||||
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: MODEL_URL,
|
||||
@@ -87,24 +53,25 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
},
|
||||
outputFaceBlendshapes: true,
|
||||
runningMode: 'VIDEO',
|
||||
numFaces: 1,
|
||||
numFaces: 3,
|
||||
minFaceDetectionConfidence: 0.65,
|
||||
minFacePresenceConfidence: 0.6,
|
||||
minTrackingConfidence: 0.6,
|
||||
});
|
||||
|
||||
if (!cancelled) {
|
||||
landmarkerRef.current = landmarker;
|
||||
setIsLoading(false);
|
||||
setIsReady(true);
|
||||
} else {
|
||||
if (cancelled) {
|
||||
landmarker.close();
|
||||
return;
|
||||
}
|
||||
|
||||
landmarkerRef.current = landmarker;
|
||||
setIsLoading(false);
|
||||
setIsReady(true);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
console.error('[MediaPipe] Initialization failed:', err);
|
||||
setError(
|
||||
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
|
||||
);
|
||||
setIsLoading(false);
|
||||
}
|
||||
if (cancelled) return;
|
||||
console.error('[MediaPipe] Initialization failed:', err);
|
||||
setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
|
||||
if (!landmarkerRef.current || !isReady) return null;
|
||||
try {
|
||||
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
|
||||
return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
/**
|
||||
* useVelocitySocket — Manages the persistent WebSocket connection to the
|
||||
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
|
||||
* routes incoming events to typed handlers, and buffers outgoing packets
|
||||
* during disconnection for flush-on-reconnect.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useCallback } from 'react';
|
||||
import { useStore } from '@/store/useStore';
|
||||
import { WS_URL } from '@/lib/api';
|
||||
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
|
||||
|
||||
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
|
||||
|
||||
// Event types coming from the backend
|
||||
type WsEventType =
|
||||
| 'WS_ASSET_OPENED'
|
||||
| 'QD_UPDATED'
|
||||
@@ -26,13 +19,9 @@ interface WsMessage {
|
||||
}
|
||||
|
||||
interface UseVelocitySocketOptions {
|
||||
/** Channel name to subscribe to (appended as a path segment) */
|
||||
channel?: 'notifications' | 'perception';
|
||||
/** Called when the socket successfully connects */
|
||||
onConnect?: () => void;
|
||||
/** Called when the socket disconnects */
|
||||
onDisconnect?: () => void;
|
||||
/** Raw message handler — bypasses the built-in notification routing */
|
||||
onMessage?: (msg: WsMessage) => void;
|
||||
}
|
||||
|
||||
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Delegate to caller's raw handler if provided
|
||||
onMessage?.(msg);
|
||||
|
||||
// Built-in notification routing
|
||||
switch (msg.type) {
|
||||
case 'WS_ASSET_OPENED': {
|
||||
const d = msg.data as Partial<VaultOpenedEvent>;
|
||||
addNotification({
|
||||
type: 'velocity_link_opened',
|
||||
title: '🚨 Velocity Link Opened',
|
||||
title: 'Velocity Link Opened',
|
||||
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
|
||||
leadId: d.lead_id,
|
||||
});
|
||||
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
if ((d.qd_score ?? 0) >= 75) {
|
||||
addNotification({
|
||||
type: 'qd_spike',
|
||||
title: '📈 QD Score Spike',
|
||||
title: 'QD Score Spike',
|
||||
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
|
||||
leadId: d.lead_id,
|
||||
qdScore: d.qd_score,
|
||||
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
if (d.tags?.length) {
|
||||
addNotification({
|
||||
type: 'lead_tagged',
|
||||
title: '🏷️ Lead Tagged',
|
||||
title: 'Lead Tagged',
|
||||
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
|
||||
leadId: d.lead_id,
|
||||
tags: d.tags,
|
||||
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
ws.onclose = () => {
|
||||
onDisconnect?.();
|
||||
if (!isMountedRef.current) return;
|
||||
// Exponential backoff: 1s, 2s, 4s … cap at 30s
|
||||
if (retryCountRef.current >= 5) return;
|
||||
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
|
||||
retryCountRef.current += 1;
|
||||
retryTimerRef.current = setTimeout(connect, delay);
|
||||
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
/** Send a JSON-serialisable payload; buffers if the socket is not open */
|
||||
const sendPacket = useCallback((payload: unknown) => {
|
||||
const str = JSON.stringify(payload);
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(str);
|
||||
} else {
|
||||
// Buffer up to 100 packets; drop oldest overflow
|
||||
pendingBufferRef.current.push(str);
|
||||
if (pendingBufferRef.current.length > 100) {
|
||||
pendingBufferRef.current.shift();
|
||||
|
||||
Reference in New Issue
Block a user