feat/#38 #44
13
app/package-lock.json
generated
13
app/package-lock.json
generated
@@ -12,6 +12,7 @@
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@mediapipe/tasks-vision": "^0.10.34",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.8",
|
||||
@@ -2668,9 +2669,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@mediapipe/tasks-vision": {
|
||||
"version": "0.10.17",
|
||||
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
|
||||
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
|
||||
"version": "0.10.34",
|
||||
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.34.tgz",
|
||||
"integrity": "sha512-KFGyhDsjJ+9WUMcMfjTOpcEp3LJNS3KwC7BfvKrCYELn/7G/5kmwnU7z6Spps+iWQoTGL8xW8i68r65OTa3DwA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@monogrid/gainmap-js": {
|
||||
@@ -4388,6 +4389,12 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": {
|
||||
"version": "0.10.17",
|
||||
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
|
||||
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@react-three/fiber": {
|
||||
"version": "9.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.5.0.tgz",
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@mediapipe/tasks-vision": "^0.10.34",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.8",
|
||||
@@ -67,8 +68,8 @@
|
||||
"zustand": "^5.0.11"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.55.0",
|
||||
"@eslint/js": "^9.39.1",
|
||||
"@playwright/test": "^1.55.0",
|
||||
"@types/node": "^24.10.1",
|
||||
"@types/react": "^19.2.5",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
|
||||
BIN
app/public/mediapipe/assets/face_landmarker.task
Normal file
BIN
app/public/mediapipe/assets/face_landmarker.task
Normal file
Binary file not shown.
8844
app/public/mediapipe/wasm/vision_wasm_internal.js
Normal file
8844
app/public/mediapipe/wasm/vision_wasm_internal.js
Normal file
File diff suppressed because it is too large
Load Diff
BIN
app/public/mediapipe/wasm/vision_wasm_internal.wasm
Normal file
BIN
app/public/mediapipe/wasm/vision_wasm_internal.wasm
Normal file
Binary file not shown.
8840
app/public/mediapipe/wasm/vision_wasm_module_internal.js
Normal file
8840
app/public/mediapipe/wasm/vision_wasm_module_internal.js
Normal file
File diff suppressed because it is too large
Load Diff
BIN
app/public/mediapipe/wasm/vision_wasm_module_internal.wasm
Normal file
BIN
app/public/mediapipe/wasm/vision_wasm_module_internal.wasm
Normal file
Binary file not shown.
8835
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.js
Normal file
8835
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.js
Normal file
File diff suppressed because it is too large
Load Diff
BIN
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.wasm
Normal file
BIN
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.wasm
Normal file
Binary file not shown.
@@ -41,7 +41,8 @@ import { useVelocitySocket } from '@/hooks/useVelocitySocket';
|
||||
import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder';
|
||||
import { API_URL } from '@/lib/api';
|
||||
import { cn } from '@/lib/utils';
|
||||
import type { QDScoreUpdate } from '@/types';
|
||||
import { useStore } from '@/store/useStore';
|
||||
import type { Lead, QDScoreUpdate } from '@/types';
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -57,6 +58,13 @@ interface PerceptionPlayerProps {
|
||||
const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const;
|
||||
type Speed = (typeof SPEEDS)[number];
|
||||
|
||||
const PERCEPTION_CAMERA_CONSTRAINTS: MediaTrackConstraints = {
|
||||
width: { ideal: 1280, min: 640 },
|
||||
height: { ideal: 720, min: 480 },
|
||||
frameRate: { ideal: 30, min: 24 },
|
||||
facingMode: 'user',
|
||||
};
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function fmt(s: number): string {
|
||||
@@ -194,6 +202,146 @@ function VolumeControl({
|
||||
|
||||
// ── Main Component ────────────────────────────────────────────────────────────
|
||||
|
||||
function clampScore(value: number): number {
|
||||
return Math.max(1, Math.min(100, Math.round(value)));
|
||||
}
|
||||
|
||||
function buildScoreMap(categories: Array<{ categoryName: string; score: number }>): Record<string, number> {
|
||||
const map: Record<string, number> = {};
|
||||
for (const category of categories) {
|
||||
map[category.categoryName] = category.score;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function deriveInitialSessionScore(lead?: Lead | null, priorInteractionCount = 0): number {
|
||||
const baseScore = lead?.quantumDynamicsScore ?? 50;
|
||||
let modifier = 0;
|
||||
const budget = (lead?.budget ?? '').toLowerCase();
|
||||
const tags = (lead?.tags ?? []).map((tag) => tag.toLowerCase());
|
||||
|
||||
if (/(10m|15m|20m|crore|million)/.test(budget)) {
|
||||
modifier += 15;
|
||||
} else if (/(5m|8m)/.test(budget)) {
|
||||
modifier += 8;
|
||||
}
|
||||
|
||||
if (tags.includes('hni')) modifier += 12;
|
||||
if (tags.includes('nri')) modifier += 5;
|
||||
|
||||
if (baseScore > 75) {
|
||||
modifier += 8;
|
||||
} else if (baseScore > 50) {
|
||||
modifier += 4;
|
||||
}
|
||||
|
||||
if (priorInteractionCount > 5) {
|
||||
modifier += 8;
|
||||
} else if (priorInteractionCount >= 2) {
|
||||
modifier += 4;
|
||||
}
|
||||
|
||||
return clampScore(baseScore + modifier);
|
||||
}
|
||||
|
||||
function calculateLocalQdScore(previousScore: number, scoreMap: Record<string, number>): number {
|
||||
const smileLeft = scoreMap.mouthSmileLeft ?? 0;
|
||||
const smileRight = scoreMap.mouthSmileRight ?? 0;
|
||||
const smileMax = Math.max(smileLeft, smileRight);
|
||||
const browInnerUp = scoreMap.browInnerUp ?? 0;
|
||||
const eyeWideLeft = scoreMap.eyeWideLeft ?? 0;
|
||||
const eyeWideRight = scoreMap.eyeWideRight ?? 0;
|
||||
const eyeWideMax = Math.max(eyeWideLeft, eyeWideRight);
|
||||
const jawOpen = scoreMap.jawOpen ?? 0;
|
||||
const cheekPuff = scoreMap.cheekPuff ?? 0;
|
||||
const browDownLeft = scoreMap.browDownLeft ?? 0;
|
||||
const browDownRight = scoreMap.browDownRight ?? 0;
|
||||
const eyeBlinkLeft = scoreMap.eyeBlinkLeft ?? 0;
|
||||
const eyeBlinkRight = scoreMap.eyeBlinkRight ?? 0;
|
||||
const mouthFrown = Math.max(scoreMap.mouthFrownLeft ?? 0, scoreMap.mouthFrownRight ?? 0);
|
||||
|
||||
let positiveDelta = 0;
|
||||
if (smileLeft > 0.5 && smileRight > 0.5) positiveDelta += 15;
|
||||
else if (smileLeft > 0.5 || smileRight > 0.5) positiveDelta += 10;
|
||||
if (browInnerUp > 0.4) positiveDelta += 8;
|
||||
if (eyeWideMax > 0.5) positiveDelta += 7;
|
||||
if (jawOpen > 0.3 && eyeWideMax > 0.5) positiveDelta += 5;
|
||||
if (cheekPuff > 0.3) positiveDelta += 3;
|
||||
|
||||
let negativeDelta = 0;
|
||||
if (browDownLeft > 0.45 && browDownRight > 0.45 && smileMax < 0.2) negativeDelta -= 10;
|
||||
if (eyeBlinkLeft > 0.7 && eyeBlinkRight > 0.7 && eyeWideMax < 0.2) negativeDelta -= 15;
|
||||
if (mouthFrown > 0.4) negativeDelta -= 8;
|
||||
|
||||
const weightedShapes = [
|
||||
smileLeft,
|
||||
smileRight,
|
||||
browInnerUp,
|
||||
eyeWideLeft,
|
||||
eyeWideRight,
|
||||
jawOpen,
|
||||
cheekPuff,
|
||||
browDownLeft,
|
||||
browDownRight,
|
||||
eyeBlinkLeft,
|
||||
eyeBlinkRight,
|
||||
mouthFrown,
|
||||
];
|
||||
if (weightedShapes.every((value) => value < 0.15)) negativeDelta -= 3;
|
||||
|
||||
let delta = positiveDelta + negativeDelta;
|
||||
if (positiveDelta > 0 && negativeDelta < 0) {
|
||||
delta = Math.abs(positiveDelta) >= Math.abs(negativeDelta) ? positiveDelta : negativeDelta;
|
||||
}
|
||||
|
||||
const boundedDelta = Math.max(-20, Math.min(20, delta));
|
||||
return clampScore(previousScore + boundedDelta);
|
||||
}
|
||||
|
||||
function getFaceBounds(
|
||||
landmarks: Array<{ x: number; y: number; z: number }>,
|
||||
containerWidth: number,
|
||||
containerHeight: number,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number,
|
||||
mirrored = true,
|
||||
) {
|
||||
const safeSourceWidth = sourceWidth > 0 ? sourceWidth : containerWidth;
|
||||
const safeSourceHeight = sourceHeight > 0 ? sourceHeight : containerHeight;
|
||||
const scale = Math.max(containerWidth / safeSourceWidth, containerHeight / safeSourceHeight);
|
||||
const renderedWidth = safeSourceWidth * scale;
|
||||
const renderedHeight = safeSourceHeight * scale;
|
||||
const offsetX = (containerWidth - renderedWidth) / 2;
|
||||
const offsetY = (containerHeight - renderedHeight) / 2;
|
||||
|
||||
let minX = Infinity;
|
||||
let minY = Infinity;
|
||||
let maxX = -Infinity;
|
||||
let maxY = -Infinity;
|
||||
|
||||
for (const point of landmarks) {
|
||||
const rawX = offsetX + (point.x * renderedWidth);
|
||||
const rawY = offsetY + (point.y * renderedHeight);
|
||||
const x = mirrored ? containerWidth - rawX : rawX;
|
||||
const y = rawY;
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x);
|
||||
maxY = Math.max(maxY, y);
|
||||
}
|
||||
|
||||
const paddingX = Math.max(8, (maxX - minX) * 0.14);
|
||||
const paddingY = Math.max(8, (maxY - minY) * 0.18);
|
||||
|
||||
return {
|
||||
x: Math.max(0, minX - paddingX),
|
||||
y: Math.max(0, minY - paddingY),
|
||||
width: Math.min(containerWidth, (maxX - minX) + (paddingX * 2)),
|
||||
height: Math.min(containerHeight, (maxY - minY) + (paddingY * 2)),
|
||||
area: Math.max(0, (maxX - minX) * (maxY - minY)),
|
||||
};
|
||||
}
|
||||
|
||||
export function PerceptionPlayer({
|
||||
videoUrl, videoAssetId, leadId,
|
||||
sessionMode = 'assigned', videoTitle, onSessionComplete,
|
||||
@@ -202,6 +350,9 @@ export function PerceptionPlayer({
|
||||
const [sessionId] = useState(
|
||||
() => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`,
|
||||
);
|
||||
const lead = useStore((state) => state.leads.find((candidate) => candidate.id === leadId) ?? null);
|
||||
const priorInteractionCount = useStore((state) => (leadId ? state.messages[leadId]?.length ?? 0 : 0));
|
||||
const initialSessionScore = deriveInitialSessionScore(lead, priorInteractionCount);
|
||||
|
||||
// Refs
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
@@ -213,13 +364,15 @@ export function PerceptionPlayer({
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const scrubbing = useRef(false);
|
||||
const qdScoreRef = useRef(initialSessionScore);
|
||||
const lastBackendQdTsRef = useRef<number>(0);
|
||||
|
||||
// State — session
|
||||
const [consentGiven, setConsentGiven] = useState(false);
|
||||
const [cameraActive, setCameraActive] = useState(false);
|
||||
const [cameraError, setCameraError] = useState<string | null>(null);
|
||||
const [sessionActive, setSessionActive] = useState(false);
|
||||
const [qdScore, setQdScore] = useState(50);
|
||||
const [qdScore, setQdScore] = useState(initialSessionScore);
|
||||
const [showCamera, setShowCamera] = useState(true);
|
||||
|
||||
// State — playback
|
||||
@@ -246,10 +399,25 @@ export function PerceptionPlayer({
|
||||
if (msg.type !== 'QD_UPDATED') return;
|
||||
const d = msg.data as Partial<QDScoreUpdate>;
|
||||
const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId;
|
||||
if (match && typeof d.qd_score === 'number') setQdScore(d.qd_score);
|
||||
if (match && typeof d.qd_score === 'number') {
|
||||
lastBackendQdTsRef.current = Date.now();
|
||||
qdScoreRef.current = d.qd_score;
|
||||
setQdScore(d.qd_score);
|
||||
}
|
||||
}, [leadId, sessionId]),
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
qdScoreRef.current = initialSessionScore;
|
||||
setQdScore(initialSessionScore);
|
||||
}, [initialSessionScore, leadId, sessionId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!cameraActive || !streamRef.current || !webcamRef.current) return;
|
||||
webcamRef.current.srcObject = streamRef.current;
|
||||
void webcamRef.current.play().catch(() => null);
|
||||
}, [cameraActive]);
|
||||
|
||||
// ── Auto-hide (YouTube behaviour) ─────────────────────────────────────────
|
||||
// Show controls, reset the hide timer. After 2.8 s of no movement while
|
||||
// playing, hide controls and the cursor.
|
||||
@@ -289,14 +457,10 @@ export function PerceptionPlayer({
|
||||
const startCamera = useCallback(async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: 320, height: 240, facingMode: 'user' },
|
||||
video: PERCEPTION_CAMERA_CONSTRAINTS,
|
||||
audio: false,
|
||||
});
|
||||
streamRef.current = stream;
|
||||
if (webcamRef.current) {
|
||||
webcamRef.current.srcObject = stream;
|
||||
await webcamRef.current.play();
|
||||
}
|
||||
setCameraActive(true);
|
||||
setCameraError(null);
|
||||
} catch (err) {
|
||||
@@ -325,39 +489,56 @@ export function PerceptionPlayer({
|
||||
|
||||
const videoTsMs = Math.round(video.currentTime * 1000);
|
||||
const result = detectFrame(webcam, now);
|
||||
if (!result?.faceBlendshapes?.length) return;
|
||||
if (!result?.faceLandmarks?.length) return;
|
||||
|
||||
const categories = result.faceBlendshapes[0]?.categories ?? [];
|
||||
const landmarkSets = result.faceLandmarks ?? [];
|
||||
const overlayWidth = canvasRef.current?.width ?? 112;
|
||||
const overlayHeight = canvasRef.current?.height ?? 80;
|
||||
const sourceWidth = webcam.videoWidth || overlayWidth;
|
||||
const sourceHeight = webcam.videoHeight || overlayHeight;
|
||||
const boundsByFace = landmarkSets.map((landmarks) =>
|
||||
getFaceBounds(landmarks, overlayWidth, overlayHeight, sourceWidth, sourceHeight, true),
|
||||
);
|
||||
const primaryFaceIndex = boundsByFace.reduce((bestIndex, bounds, index, allBounds) => (
|
||||
bounds.area > allBounds[bestIndex].area ? index : bestIndex
|
||||
), 0);
|
||||
|
||||
const categories = result.faceBlendshapes?.[primaryFaceIndex]?.categories ?? [];
|
||||
const localScoreMap = buildScoreMap(categories);
|
||||
const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId);
|
||||
if (!hasSignificantActivity(packet)) return;
|
||||
const significantActivity = hasSignificantActivity(packet);
|
||||
|
||||
sendPacket({
|
||||
event: 'BIOMETRIC_PACKET',
|
||||
lead_id: leadId, session_id: sessionId,
|
||||
session_mode: sessionMode, video_asset_id: videoAssetId,
|
||||
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
|
||||
});
|
||||
if (significantActivity) {
|
||||
sendPacket({
|
||||
event: 'BIOMETRIC_PACKET',
|
||||
lead_id: leadId, session_id: sessionId,
|
||||
session_mode: sessionMode, video_asset_id: videoAssetId,
|
||||
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
|
||||
});
|
||||
}
|
||||
|
||||
if (Date.now() - lastBackendQdTsRef.current > 2500) {
|
||||
const rawScore = calculateLocalQdScore(qdScoreRef.current, localScoreMap);
|
||||
const smoothedScore = clampScore((qdScoreRef.current * 0.7) + (rawScore * 0.3));
|
||||
qdScoreRef.current = smoothedScore;
|
||||
setQdScore(smoothedScore);
|
||||
}
|
||||
|
||||
const canvas = canvasRef.current;
|
||||
if (canvas && result.faceLandmarks?.length) {
|
||||
if (canvas && landmarkSets.length) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (ctx) {
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
const lm = result.faceLandmarks[0];
|
||||
if (lm?.length) {
|
||||
const pts = [10, 234, 454, 152, 58, 288];
|
||||
ctx.strokeStyle = 'rgba(59,130,246,0.7)';
|
||||
ctx.lineWidth = 1.5;
|
||||
boundsByFace.forEach((bounds, index) => {
|
||||
const isPrimary = index === primaryFaceIndex;
|
||||
ctx.strokeStyle = isPrimary ? 'rgba(59,130,246,0.92)' : 'rgba(255,255,255,0.5)';
|
||||
ctx.lineWidth = isPrimary ? 2 : 1.2;
|
||||
ctx.fillStyle = isPrimary ? 'rgba(59,130,246,0.12)' : 'rgba(255,255,255,0.06)';
|
||||
ctx.beginPath();
|
||||
pts.forEach((idx, i) => {
|
||||
const p = lm[idx];
|
||||
if (!p) return;
|
||||
i === 0 ? ctx.moveTo(p.x * canvas.width, p.y * canvas.height)
|
||||
: ctx.lineTo(p.x * canvas.width, p.y * canvas.height);
|
||||
});
|
||||
ctx.closePath();
|
||||
ctx.roundRect(bounds.x, bounds.y, bounds.width, bounds.height, 8);
|
||||
ctx.fill();
|
||||
ctx.stroke();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]);
|
||||
|
||||
1
app/src/global.d.ts
vendored
1
app/src/global.d.ts
vendored
@@ -1 +0,0 @@
|
||||
declare module 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs';
|
||||
@@ -1,27 +1,8 @@
|
||||
/**
|
||||
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
|
||||
* FaceLandmarker WASM task and exposes a frame-processing callback.
|
||||
*
|
||||
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
|
||||
* CDN path). Initialization is async; `isReady` is false until complete.
|
||||
*
|
||||
* Usage:
|
||||
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
|
||||
* // In rAF loop:
|
||||
* const result = detectFrame(videoElement, performance.now());
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useRef, useCallback } from 'react';
|
||||
|
||||
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
|
||||
// Types are inlined below so we don't need @types/mediapipe
|
||||
type FaceLandmarker = {
|
||||
detectForVideo: (
|
||||
videoElement: HTMLVideoElement,
|
||||
timestamp: number,
|
||||
) => FaceLandmarkerResult;
|
||||
close: () => void;
|
||||
};
|
||||
import {
|
||||
FaceLandmarker,
|
||||
FilesetResolver,
|
||||
} from '@mediapipe/tasks-vision';
|
||||
|
||||
export interface BlendShapeCategory {
|
||||
categoryName: string;
|
||||
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
|
||||
|
||||
const MODEL_URL =
|
||||
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
|
||||
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
|
||||
'/mediapipe/assets/face_landmarker.task';
|
||||
|
||||
const WASM_ROOT =
|
||||
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
|
||||
'/mediapipe/wasm';
|
||||
|
||||
interface UseFaceLandmarkerReturn {
|
||||
isLoading: boolean;
|
||||
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
// Dynamic import — avoids pulling the WASM runtime into the main bundle
|
||||
const vision = await import(
|
||||
/* @vite-ignore */
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
|
||||
);
|
||||
|
||||
const { FaceLandmarker, FilesetResolver } = vision as {
|
||||
FaceLandmarker: {
|
||||
createFromOptions: (
|
||||
resolver: unknown,
|
||||
options: unknown,
|
||||
) => Promise<FaceLandmarker>;
|
||||
};
|
||||
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
|
||||
};
|
||||
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
|
||||
);
|
||||
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
|
||||
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: MODEL_URL,
|
||||
@@ -87,24 +53,25 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
},
|
||||
outputFaceBlendshapes: true,
|
||||
runningMode: 'VIDEO',
|
||||
numFaces: 1,
|
||||
numFaces: 3,
|
||||
minFaceDetectionConfidence: 0.65,
|
||||
minFacePresenceConfidence: 0.6,
|
||||
minTrackingConfidence: 0.6,
|
||||
});
|
||||
|
||||
if (!cancelled) {
|
||||
landmarkerRef.current = landmarker;
|
||||
setIsLoading(false);
|
||||
setIsReady(true);
|
||||
} else {
|
||||
if (cancelled) {
|
||||
landmarker.close();
|
||||
return;
|
||||
}
|
||||
|
||||
landmarkerRef.current = landmarker;
|
||||
setIsLoading(false);
|
||||
setIsReady(true);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
console.error('[MediaPipe] Initialization failed:', err);
|
||||
setError(
|
||||
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
|
||||
);
|
||||
setIsLoading(false);
|
||||
}
|
||||
if (cancelled) return;
|
||||
console.error('[MediaPipe] Initialization failed:', err);
|
||||
setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
|
||||
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
|
||||
if (!landmarkerRef.current || !isReady) return null;
|
||||
try {
|
||||
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
|
||||
return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
/**
|
||||
* useVelocitySocket — Manages the persistent WebSocket connection to the
|
||||
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
|
||||
* routes incoming events to typed handlers, and buffers outgoing packets
|
||||
* during disconnection for flush-on-reconnect.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef, useCallback } from 'react';
|
||||
import { useStore } from '@/store/useStore';
|
||||
import { WS_URL } from '@/lib/api';
|
||||
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
|
||||
|
||||
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
|
||||
|
||||
// Event types coming from the backend
|
||||
type WsEventType =
|
||||
| 'WS_ASSET_OPENED'
|
||||
| 'QD_UPDATED'
|
||||
@@ -26,13 +19,9 @@ interface WsMessage {
|
||||
}
|
||||
|
||||
interface UseVelocitySocketOptions {
|
||||
/** Channel name to subscribe to (appended as a path segment) */
|
||||
channel?: 'notifications' | 'perception';
|
||||
/** Called when the socket successfully connects */
|
||||
onConnect?: () => void;
|
||||
/** Called when the socket disconnects */
|
||||
onDisconnect?: () => void;
|
||||
/** Raw message handler — bypasses the built-in notification routing */
|
||||
onMessage?: (msg: WsMessage) => void;
|
||||
}
|
||||
|
||||
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Delegate to caller's raw handler if provided
|
||||
onMessage?.(msg);
|
||||
|
||||
// Built-in notification routing
|
||||
switch (msg.type) {
|
||||
case 'WS_ASSET_OPENED': {
|
||||
const d = msg.data as Partial<VaultOpenedEvent>;
|
||||
addNotification({
|
||||
type: 'velocity_link_opened',
|
||||
title: '🚨 Velocity Link Opened',
|
||||
title: 'Velocity Link Opened',
|
||||
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
|
||||
leadId: d.lead_id,
|
||||
});
|
||||
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
if ((d.qd_score ?? 0) >= 75) {
|
||||
addNotification({
|
||||
type: 'qd_spike',
|
||||
title: '📈 QD Score Spike',
|
||||
title: 'QD Score Spike',
|
||||
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
|
||||
leadId: d.lead_id,
|
||||
qdScore: d.qd_score,
|
||||
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
if (d.tags?.length) {
|
||||
addNotification({
|
||||
type: 'lead_tagged',
|
||||
title: '🏷️ Lead Tagged',
|
||||
title: 'Lead Tagged',
|
||||
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
|
||||
leadId: d.lead_id,
|
||||
tags: d.tags,
|
||||
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
ws.onclose = () => {
|
||||
onDisconnect?.();
|
||||
if (!isMountedRef.current) return;
|
||||
// Exponential backoff: 1s, 2s, 4s … cap at 30s
|
||||
if (retryCountRef.current >= 5) return;
|
||||
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
|
||||
retryCountRef.current += 1;
|
||||
retryTimerRef.current = setTimeout(connect, delay);
|
||||
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
/** Send a JSON-serialisable payload; buffers if the socket is not open */
|
||||
const sendPacket = useCallback((payload: unknown) => {
|
||||
const str = JSON.stringify(payload);
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(str);
|
||||
} else {
|
||||
// Buffer up to 100 packets; drop oldest overflow
|
||||
pendingBufferRef.current.push(str);
|
||||
if (pendingBufferRef.current.length > 100) {
|
||||
pendingBufferRef.current.shift();
|
||||
|
||||
Reference in New Issue
Block a user