Co-authored-by: Sagnik <sagnik7896@gmail.com>
Reviewed-on: #23
This commit was merged in pull request #23.
This commit is contained in:
2026-04-13 02:35:24 +05:30
parent bf950bc789
commit 857e0b88e6
13 changed files with 26773 additions and 114 deletions

View File

@@ -1,27 +1,8 @@
/**
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
* FaceLandmarker WASM task and exposes a frame-processing callback.
*
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
* CDN path). Initialization is async; `isReady` is false until complete.
*
* Usage:
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
* // In rAF loop:
* const result = detectFrame(videoElement, performance.now());
*/
import { useState, useEffect, useRef, useCallback } from 'react';
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
// Types are inlined below so we don't need @types/mediapipe
type FaceLandmarker = {
detectForVideo: (
videoElement: HTMLVideoElement,
timestamp: number,
) => FaceLandmarkerResult;
close: () => void;
};
import {
FaceLandmarker,
FilesetResolver,
} from '@mediapipe/tasks-vision';
export interface BlendShapeCategory {
categoryName: string;
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
const MODEL_URL =
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
'/mediapipe/assets/face_landmarker.task';
const WASM_ROOT =
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
'/mediapipe/wasm';
interface UseFaceLandmarkerReturn {
isLoading: boolean;
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
async function init() {
try {
// Dynamic import — avoids pulling the WASM runtime into the main bundle
const vision = await import(
/* @vite-ignore */
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
);
const { FaceLandmarker, FilesetResolver } = vision as {
FaceLandmarker: {
createFromOptions: (
resolver: unknown,
options: unknown,
) => Promise<FaceLandmarker>;
};
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
};
const filesetResolver = await FilesetResolver.forVisionTasks(
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
);
const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: {
modelAssetPath: MODEL_URL,
@@ -87,24 +53,25 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
},
outputFaceBlendshapes: true,
runningMode: 'VIDEO',
numFaces: 1,
numFaces: 3,
minFaceDetectionConfidence: 0.65,
minFacePresenceConfidence: 0.6,
minTrackingConfidence: 0.6,
});
if (!cancelled) {
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} else {
if (cancelled) {
landmarker.close();
return;
}
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} catch (err) {
if (!cancelled) {
console.error('[MediaPipe] Initialization failed:', err);
setError(
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
);
setIsLoading(false);
}
if (cancelled) return;
console.error('[MediaPipe] Initialization failed:', err);
setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
setIsLoading(false);
}
}
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
if (!landmarkerRef.current || !isReady) return null;
try {
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
} catch {
return null;
}

View File

@@ -1,17 +1,10 @@
/**
* useVelocitySocket — Manages the persistent WebSocket connection to the
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
* routes incoming events to typed handlers, and buffers outgoing packets
* during disconnection for flush-on-reconnect.
*/
import { useEffect, useRef, useCallback } from 'react';
import { useStore } from '@/store/useStore';
import { WS_URL } from '@/lib/api';
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
// Event types coming from the backend
type WsEventType =
| 'WS_ASSET_OPENED'
| 'QD_UPDATED'
@@ -26,13 +19,9 @@ interface WsMessage {
}
interface UseVelocitySocketOptions {
/** Channel name to subscribe to (appended as a path segment) */
channel?: 'notifications' | 'perception';
/** Called when the socket successfully connects */
onConnect?: () => void;
/** Called when the socket disconnects */
onDisconnect?: () => void;
/** Raw message handler — bypasses the built-in notification routing */
onMessage?: (msg: WsMessage) => void;
}
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
return;
}
// Delegate to caller's raw handler if provided
onMessage?.(msg);
// Built-in notification routing
switch (msg.type) {
case 'WS_ASSET_OPENED': {
const d = msg.data as Partial<VaultOpenedEvent>;
addNotification({
type: 'velocity_link_opened',
title: '🚨 Velocity Link Opened',
title: 'Velocity Link Opened',
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
leadId: d.lead_id,
});
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if ((d.qd_score ?? 0) >= 75) {
addNotification({
type: 'qd_spike',
title: '📈 QD Score Spike',
title: 'QD Score Spike',
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
leadId: d.lead_id,
qdScore: d.qd_score,
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if (d.tags?.length) {
addNotification({
type: 'lead_tagged',
title: '🏷️ Lead Tagged',
title: 'Lead Tagged',
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
leadId: d.lead_id,
tags: d.tags,
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
ws.onclose = () => {
onDisconnect?.();
if (!isMountedRef.current) return;
// Exponential backoff: 1s, 2s, 4s … cap at 30s
if (retryCountRef.current >= 5) return;
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
retryCountRef.current += 1;
retryTimerRef.current = setTimeout(connect, delay);
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
};
}, [connect]);
/** Send a JSON-serialisable payload; buffers if the socket is not open */
const sendPacket = useCallback((payload: unknown) => {
const str = JSON.stringify(payload);
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(str);
} else {
// Buffer up to 100 packets; drop oldest overflow
pendingBufferRef.current.push(str);
if (pendingBufferRef.current.length > 100) {
pendingBufferRef.current.shift();