Built the Sentinel Tab
This commit is contained in:
Sagnik
133
app/src/hooks/useMediapipeFaceLandmarker.ts
Normal file
133
app/src/hooks/useMediapipeFaceLandmarker.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
|
||||
* FaceLandmarker WASM task and exposes a frame-processing callback.
|
||||
*
|
||||
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
|
||||
* CDN path). Initialization is async; `isReady` is false until complete.
|
||||
*
|
||||
* Usage:
|
||||
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
|
||||
* // In rAF loop:
|
||||
* const result = detectFrame(videoElement, performance.now());
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useRef, useCallback } from 'react';
|
||||
|
||||
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
|
||||
// Types are inlined below so we don't need @types/mediapipe
|
||||
type FaceLandmarker = {
|
||||
detectForVideo: (
|
||||
videoElement: HTMLVideoElement,
|
||||
timestamp: number,
|
||||
) => FaceLandmarkerResult;
|
||||
close: () => void;
|
||||
};
|
||||
|
||||
export interface BlendShapeCategory {
|
||||
categoryName: string;
|
||||
score: number;
|
||||
displayName: string;
|
||||
index: number;
|
||||
}
|
||||
|
||||
export interface FaceLandmarkerResult {
|
||||
faceBlendshapes: Array<{ categories: BlendShapeCategory[] }>;
|
||||
faceLandmarks: Array<Array<{ x: number; y: number; z: number }>>;
|
||||
}
|
||||
|
||||
const MODEL_URL =
|
||||
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
|
||||
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
|
||||
|
||||
interface UseFaceLandmarkerReturn {
|
||||
isLoading: boolean;
|
||||
isReady: boolean;
|
||||
error: string | null;
|
||||
detectFrame: (
|
||||
videoElement: HTMLVideoElement,
|
||||
timestampMs: number,
|
||||
) => FaceLandmarkerResult | null;
|
||||
}
|
||||
|
||||
export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
const landmarkerRef = useRef<FaceLandmarker | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [isReady, setIsReady] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
// Dynamic import — avoids pulling the WASM runtime into the main bundle
|
||||
const vision = await import(
|
||||
/* @vite-ignore */
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
|
||||
);
|
||||
|
||||
const { FaceLandmarker, FilesetResolver } = vision as {
|
||||
FaceLandmarker: {
|
||||
createFromOptions: (
|
||||
resolver: unknown,
|
||||
options: unknown,
|
||||
) => Promise<FaceLandmarker>;
|
||||
};
|
||||
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
|
||||
};
|
||||
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
|
||||
);
|
||||
|
||||
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: MODEL_URL,
|
||||
delegate: 'GPU',
|
||||
},
|
||||
outputFaceBlendshapes: true,
|
||||
runningMode: 'VIDEO',
|
||||
numFaces: 1,
|
||||
});
|
||||
|
||||
if (!cancelled) {
|
||||
landmarkerRef.current = landmarker;
|
||||
setIsLoading(false);
|
||||
setIsReady(true);
|
||||
} else {
|
||||
landmarker.close();
|
||||
}
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
console.error('[MediaPipe] Initialization failed:', err);
|
||||
setError(
|
||||
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
|
||||
);
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void init();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
landmarkerRef.current?.close();
|
||||
landmarkerRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const detectFrame = useCallback(
|
||||
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
|
||||
if (!landmarkerRef.current || !isReady) return null;
|
||||
try {
|
||||
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
[isReady],
|
||||
);
|
||||
|
||||
return { isLoading, isReady, error, detectFrame };
|
||||
}
|
||||
160
app/src/hooks/useVelocitySocket.ts
Normal file
160
app/src/hooks/useVelocitySocket.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* useVelocitySocket — Manages the persistent WebSocket connection to the
|
||||
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
|
||||
* routes incoming events to typed handlers, and buffers outgoing packets
|
||||
* during disconnection for flush-on-reconnect.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useCallback } from 'react';
|
||||
import { useStore } from '@/store/useStore';
|
||||
import { WS_URL } from '@/lib/api';
|
||||
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
|
||||
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
|
||||
|
||||
// Event types coming from the backend
|
||||
type WsEventType =
|
||||
| 'WS_ASSET_OPENED'
|
||||
| 'QD_UPDATED'
|
||||
| 'LEAD_TAGGED'
|
||||
| 'system'
|
||||
| 'ack';
|
||||
|
||||
interface WsMessage {
|
||||
type: WsEventType;
|
||||
data?: Record<string, unknown>;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface UseVelocitySocketOptions {
|
||||
/** Channel name to subscribe to (appended as a path segment) */
|
||||
channel?: 'notifications' | 'perception';
|
||||
/** Called when the socket successfully connects */
|
||||
onConnect?: () => void;
|
||||
/** Called when the socket disconnects */
|
||||
onDisconnect?: () => void;
|
||||
/** Raw message handler — bypasses the built-in notification routing */
|
||||
onMessage?: (msg: WsMessage) => void;
|
||||
}
|
||||
|
||||
export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
const { channel = 'notifications', onConnect, onDisconnect, onMessage } = options;
|
||||
const { addNotification } = useStore();
|
||||
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const retryCountRef = useRef(0);
|
||||
const retryTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const pendingBufferRef = useRef<string[]>([]);
|
||||
const isMountedRef = useRef(true);
|
||||
|
||||
const handleMessage = useCallback(
|
||||
(event: MessageEvent) => {
|
||||
let msg: WsMessage;
|
||||
try {
|
||||
msg = JSON.parse(event.data as string) as WsMessage;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
// Delegate to caller's raw handler if provided
|
||||
onMessage?.(msg);
|
||||
|
||||
// Built-in notification routing
|
||||
switch (msg.type) {
|
||||
case 'WS_ASSET_OPENED': {
|
||||
const d = msg.data as Partial<VaultOpenedEvent>;
|
||||
addNotification({
|
||||
type: 'velocity_link_opened',
|
||||
title: '🚨 Velocity Link Opened',
|
||||
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
|
||||
leadId: d.lead_id,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'QD_UPDATED': {
|
||||
const d = msg.data as Partial<QDScoreUpdate>;
|
||||
if ((d.qd_score ?? 0) >= 75) {
|
||||
addNotification({
|
||||
type: 'qd_spike',
|
||||
title: '📈 QD Score Spike',
|
||||
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
|
||||
leadId: d.lead_id,
|
||||
qdScore: d.qd_score,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'LEAD_TAGGED': {
|
||||
const d = msg.data as { lead_id?: string; lead_name?: string; tags?: string[] };
|
||||
if (d.tags?.length) {
|
||||
addNotification({
|
||||
type: 'lead_tagged',
|
||||
title: '🏷️ Lead Tagged',
|
||||
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
|
||||
leadId: d.lead_id,
|
||||
tags: d.tags,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
},
|
||||
[addNotification, onMessage],
|
||||
);
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (!isMountedRef.current) return;
|
||||
|
||||
const url = `${SENTINEL_WS_ROOT}/${channel}`;
|
||||
const ws = new WebSocket(url);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.onopen = () => {
|
||||
retryCountRef.current = 0;
|
||||
pendingBufferRef.current.forEach((msg) => ws.send(msg));
|
||||
pendingBufferRef.current = [];
|
||||
onConnect?.();
|
||||
};
|
||||
|
||||
ws.onmessage = handleMessage;
|
||||
|
||||
ws.onclose = () => {
|
||||
onDisconnect?.();
|
||||
if (!isMountedRef.current) return;
|
||||
// Exponential backoff: 1s, 2s, 4s … cap at 30s
|
||||
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
|
||||
retryCountRef.current += 1;
|
||||
retryTimerRef.current = setTimeout(connect, delay);
|
||||
};
|
||||
|
||||
ws.onerror = () => ws.close();
|
||||
}, [channel, handleMessage, onConnect, onDisconnect]);
|
||||
|
||||
useEffect(() => {
|
||||
isMountedRef.current = true;
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
isMountedRef.current = false;
|
||||
if (retryTimerRef.current) clearTimeout(retryTimerRef.current);
|
||||
wsRef.current?.close();
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
/** Send a JSON-serialisable payload; buffers if the socket is not open */
|
||||
const sendPacket = useCallback((payload: unknown) => {
|
||||
const str = JSON.stringify(payload);
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(str);
|
||||
} else {
|
||||
// Buffer up to 100 packets; drop oldest overflow
|
||||
pendingBufferRef.current.push(str);
|
||||
if (pendingBufferRef.current.length > 100) {
|
||||
pendingBufferRef.current.shift();
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
return { sendPacket };
|
||||
}
|
||||
Reference in New Issue
Block a user