merge upstream

This commit is contained in:
2026-04-18 16:57:32 +05:30
13 changed files with 26773 additions and 114 deletions

13
app/package-lock.json generated
View File

@@ -12,6 +12,7 @@
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@hookform/resolvers": "^5.2.2",
"@mediapipe/tasks-vision": "^0.10.34",
"@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.8",
@@ -2668,9 +2669,9 @@
}
},
"node_modules/@mediapipe/tasks-vision": {
"version": "0.10.17",
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
"version": "0.10.34",
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.34.tgz",
"integrity": "sha512-KFGyhDsjJ+9WUMcMfjTOpcEp3LJNS3KwC7BfvKrCYELn/7G/5kmwnU7z6Spps+iWQoTGL8xW8i68r65OTa3DwA==",
"license": "Apache-2.0"
},
"node_modules/@monogrid/gainmap-js": {
@@ -4388,6 +4389,12 @@
}
}
},
"node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": {
"version": "0.10.17",
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
"license": "Apache-2.0"
},
"node_modules/@react-three/fiber": {
"version": "9.5.0",
"resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.5.0.tgz",

View File

@@ -15,6 +15,7 @@
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@hookform/resolvers": "^5.2.2",
"@mediapipe/tasks-vision": "^0.10.34",
"@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.8",
@@ -67,8 +68,8 @@
"zustand": "^5.0.11"
},
"devDependencies": {
"@playwright/test": "^1.55.0",
"@eslint/js": "^9.39.1",
"@playwright/test": "^1.55.0",
"@types/node": "^24.10.1",
"@types/react": "^19.2.5",
"@types/react-dom": "^19.2.3",

Binary file not shown.

File diff suppressed because it is too large Load Diff

Binary file not shown.

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -41,7 +41,8 @@ import { useVelocitySocket } from '@/hooks/useVelocitySocket';
import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder';
import { API_URL } from '@/lib/api';
import { cn } from '@/lib/utils';
import type { QDScoreUpdate } from '@/types';
import { useStore } from '@/store/useStore';
import type { Lead, QDScoreUpdate } from '@/types';
// ── Types ─────────────────────────────────────────────────────────────────────
@@ -57,6 +58,13 @@ interface PerceptionPlayerProps {
const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const;
type Speed = (typeof SPEEDS)[number];
const PERCEPTION_CAMERA_CONSTRAINTS: MediaTrackConstraints = {
width: { ideal: 1280, min: 640 },
height: { ideal: 720, min: 480 },
frameRate: { ideal: 30, min: 24 },
facingMode: 'user',
};
// ── Helpers ───────────────────────────────────────────────────────────────────
function fmt(s: number): string {
@@ -194,6 +202,146 @@ function VolumeControl({
// ── Main Component ────────────────────────────────────────────────────────────
function clampScore(value: number): number {
return Math.max(1, Math.min(100, Math.round(value)));
}
function buildScoreMap(categories: Array<{ categoryName: string; score: number }>): Record<string, number> {
const map: Record<string, number> = {};
for (const category of categories) {
map[category.categoryName] = category.score;
}
return map;
}
function deriveInitialSessionScore(lead?: Lead | null, priorInteractionCount = 0): number {
const baseScore = lead?.quantumDynamicsScore ?? 50;
let modifier = 0;
const budget = (lead?.budget ?? '').toLowerCase();
const tags = (lead?.tags ?? []).map((tag) => tag.toLowerCase());
if (/(10m|15m|20m|crore|million)/.test(budget)) {
modifier += 15;
} else if (/(5m|8m)/.test(budget)) {
modifier += 8;
}
if (tags.includes('hni')) modifier += 12;
if (tags.includes('nri')) modifier += 5;
if (baseScore > 75) {
modifier += 8;
} else if (baseScore > 50) {
modifier += 4;
}
if (priorInteractionCount > 5) {
modifier += 8;
} else if (priorInteractionCount >= 2) {
modifier += 4;
}
return clampScore(baseScore + modifier);
}
function calculateLocalQdScore(previousScore: number, scoreMap: Record<string, number>): number {
const smileLeft = scoreMap.mouthSmileLeft ?? 0;
const smileRight = scoreMap.mouthSmileRight ?? 0;
const smileMax = Math.max(smileLeft, smileRight);
const browInnerUp = scoreMap.browInnerUp ?? 0;
const eyeWideLeft = scoreMap.eyeWideLeft ?? 0;
const eyeWideRight = scoreMap.eyeWideRight ?? 0;
const eyeWideMax = Math.max(eyeWideLeft, eyeWideRight);
const jawOpen = scoreMap.jawOpen ?? 0;
const cheekPuff = scoreMap.cheekPuff ?? 0;
const browDownLeft = scoreMap.browDownLeft ?? 0;
const browDownRight = scoreMap.browDownRight ?? 0;
const eyeBlinkLeft = scoreMap.eyeBlinkLeft ?? 0;
const eyeBlinkRight = scoreMap.eyeBlinkRight ?? 0;
const mouthFrown = Math.max(scoreMap.mouthFrownLeft ?? 0, scoreMap.mouthFrownRight ?? 0);
let positiveDelta = 0;
if (smileLeft > 0.5 && smileRight > 0.5) positiveDelta += 15;
else if (smileLeft > 0.5 || smileRight > 0.5) positiveDelta += 10;
if (browInnerUp > 0.4) positiveDelta += 8;
if (eyeWideMax > 0.5) positiveDelta += 7;
if (jawOpen > 0.3 && eyeWideMax > 0.5) positiveDelta += 5;
if (cheekPuff > 0.3) positiveDelta += 3;
let negativeDelta = 0;
if (browDownLeft > 0.45 && browDownRight > 0.45 && smileMax < 0.2) negativeDelta -= 10;
if (eyeBlinkLeft > 0.7 && eyeBlinkRight > 0.7 && eyeWideMax < 0.2) negativeDelta -= 15;
if (mouthFrown > 0.4) negativeDelta -= 8;
const weightedShapes = [
smileLeft,
smileRight,
browInnerUp,
eyeWideLeft,
eyeWideRight,
jawOpen,
cheekPuff,
browDownLeft,
browDownRight,
eyeBlinkLeft,
eyeBlinkRight,
mouthFrown,
];
if (weightedShapes.every((value) => value < 0.15)) negativeDelta -= 3;
let delta = positiveDelta + negativeDelta;
if (positiveDelta > 0 && negativeDelta < 0) {
delta = Math.abs(positiveDelta) >= Math.abs(negativeDelta) ? positiveDelta : negativeDelta;
}
const boundedDelta = Math.max(-20, Math.min(20, delta));
return clampScore(previousScore + boundedDelta);
}
function getFaceBounds(
landmarks: Array<{ x: number; y: number; z: number }>,
containerWidth: number,
containerHeight: number,
sourceWidth: number,
sourceHeight: number,
mirrored = true,
) {
const safeSourceWidth = sourceWidth > 0 ? sourceWidth : containerWidth;
const safeSourceHeight = sourceHeight > 0 ? sourceHeight : containerHeight;
const scale = Math.max(containerWidth / safeSourceWidth, containerHeight / safeSourceHeight);
const renderedWidth = safeSourceWidth * scale;
const renderedHeight = safeSourceHeight * scale;
const offsetX = (containerWidth - renderedWidth) / 2;
const offsetY = (containerHeight - renderedHeight) / 2;
let minX = Infinity;
let minY = Infinity;
let maxX = -Infinity;
let maxY = -Infinity;
for (const point of landmarks) {
const rawX = offsetX + (point.x * renderedWidth);
const rawY = offsetY + (point.y * renderedHeight);
const x = mirrored ? containerWidth - rawX : rawX;
const y = rawY;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
const paddingX = Math.max(8, (maxX - minX) * 0.14);
const paddingY = Math.max(8, (maxY - minY) * 0.18);
return {
x: Math.max(0, minX - paddingX),
y: Math.max(0, minY - paddingY),
width: Math.min(containerWidth, (maxX - minX) + (paddingX * 2)),
height: Math.min(containerHeight, (maxY - minY) + (paddingY * 2)),
area: Math.max(0, (maxX - minX) * (maxY - minY)),
};
}
export function PerceptionPlayer({
videoUrl, videoAssetId, leadId,
sessionMode = 'assigned', videoTitle, onSessionComplete,
@@ -202,6 +350,9 @@ export function PerceptionPlayer({
const [sessionId] = useState(
() => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`,
);
const lead = useStore((state) => state.leads.find((candidate) => candidate.id === leadId) ?? null);
const priorInteractionCount = useStore((state) => (leadId ? state.messages[leadId]?.length ?? 0 : 0));
const initialSessionScore = deriveInitialSessionScore(lead, priorInteractionCount);
// Refs
const videoRef = useRef<HTMLVideoElement>(null);
@@ -213,13 +364,15 @@ export function PerceptionPlayer({
const streamRef = useRef<MediaStream | null>(null);
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const scrubbing = useRef(false);
const qdScoreRef = useRef(initialSessionScore);
const lastBackendQdTsRef = useRef<number>(0);
// State — session
const [consentGiven, setConsentGiven] = useState(false);
const [cameraActive, setCameraActive] = useState(false);
const [cameraError, setCameraError] = useState<string | null>(null);
const [sessionActive, setSessionActive] = useState(false);
const [qdScore, setQdScore] = useState(50);
const [qdScore, setQdScore] = useState(initialSessionScore);
const [showCamera, setShowCamera] = useState(true);
// State — playback
@@ -246,10 +399,25 @@ export function PerceptionPlayer({
if (msg.type !== 'QD_UPDATED') return;
const d = msg.data as Partial<QDScoreUpdate>;
const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId;
if (match && typeof d.qd_score === 'number') setQdScore(d.qd_score);
if (match && typeof d.qd_score === 'number') {
lastBackendQdTsRef.current = Date.now();
qdScoreRef.current = d.qd_score;
setQdScore(d.qd_score);
}
}, [leadId, sessionId]),
});
useEffect(() => {
qdScoreRef.current = initialSessionScore;
setQdScore(initialSessionScore);
}, [initialSessionScore, leadId, sessionId]);
useEffect(() => {
if (!cameraActive || !streamRef.current || !webcamRef.current) return;
webcamRef.current.srcObject = streamRef.current;
void webcamRef.current.play().catch(() => null);
}, [cameraActive]);
// ── Auto-hide (YouTube behaviour) ─────────────────────────────────────────
// Show controls, reset the hide timer. After 2.8 s of no movement while
// playing, hide controls and the cursor.
@@ -289,14 +457,10 @@ export function PerceptionPlayer({
const startCamera = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' },
video: PERCEPTION_CAMERA_CONSTRAINTS,
audio: false,
});
streamRef.current = stream;
if (webcamRef.current) {
webcamRef.current.srcObject = stream;
await webcamRef.current.play();
}
setCameraActive(true);
setCameraError(null);
} catch (err) {
@@ -325,39 +489,56 @@ export function PerceptionPlayer({
const videoTsMs = Math.round(video.currentTime * 1000);
const result = detectFrame(webcam, now);
if (!result?.faceBlendshapes?.length) return;
if (!result?.faceLandmarks?.length) return;
const categories = result.faceBlendshapes[0]?.categories ?? [];
const landmarkSets = result.faceLandmarks ?? [];
const overlayWidth = canvasRef.current?.width ?? 112;
const overlayHeight = canvasRef.current?.height ?? 80;
const sourceWidth = webcam.videoWidth || overlayWidth;
const sourceHeight = webcam.videoHeight || overlayHeight;
const boundsByFace = landmarkSets.map((landmarks) =>
getFaceBounds(landmarks, overlayWidth, overlayHeight, sourceWidth, sourceHeight, true),
);
const primaryFaceIndex = boundsByFace.reduce((bestIndex, bounds, index, allBounds) => (
bounds.area > allBounds[bestIndex].area ? index : bestIndex
), 0);
const categories = result.faceBlendshapes?.[primaryFaceIndex]?.categories ?? [];
const localScoreMap = buildScoreMap(categories);
const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId);
if (!hasSignificantActivity(packet)) return;
const significantActivity = hasSignificantActivity(packet);
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
if (significantActivity) {
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
}
if (Date.now() - lastBackendQdTsRef.current > 2500) {
const rawScore = calculateLocalQdScore(qdScoreRef.current, localScoreMap);
const smoothedScore = clampScore((qdScoreRef.current * 0.7) + (rawScore * 0.3));
qdScoreRef.current = smoothedScore;
setQdScore(smoothedScore);
}
const canvas = canvasRef.current;
if (canvas && result.faceLandmarks?.length) {
if (canvas && landmarkSets.length) {
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
const lm = result.faceLandmarks[0];
if (lm?.length) {
const pts = [10, 234, 454, 152, 58, 288];
ctx.strokeStyle = 'rgba(59,130,246,0.7)';
ctx.lineWidth = 1.5;
boundsByFace.forEach((bounds, index) => {
const isPrimary = index === primaryFaceIndex;
ctx.strokeStyle = isPrimary ? 'rgba(59,130,246,0.92)' : 'rgba(255,255,255,0.5)';
ctx.lineWidth = isPrimary ? 2 : 1.2;
ctx.fillStyle = isPrimary ? 'rgba(59,130,246,0.12)' : 'rgba(255,255,255,0.06)';
ctx.beginPath();
pts.forEach((idx, i) => {
const p = lm[idx];
if (!p) return;
i === 0 ? ctx.moveTo(p.x * canvas.width, p.y * canvas.height)
: ctx.lineTo(p.x * canvas.width, p.y * canvas.height);
});
ctx.closePath();
ctx.roundRect(bounds.x, bounds.y, bounds.width, bounds.height, 8);
ctx.fill();
ctx.stroke();
}
});
}
}
}, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]);

1
app/src/global.d.ts vendored
View File

@@ -1 +0,0 @@
declare module 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs';

View File

@@ -1,27 +1,8 @@
/**
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
* FaceLandmarker WASM task and exposes a frame-processing callback.
*
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
* CDN path). Initialization is async; `isReady` is false until complete.
*
* Usage:
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
* // In rAF loop:
* const result = detectFrame(videoElement, performance.now());
*/
import { useState, useEffect, useRef, useCallback } from 'react';
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
// Types are inlined below so we don't need @types/mediapipe
type FaceLandmarker = {
detectForVideo: (
videoElement: HTMLVideoElement,
timestamp: number,
) => FaceLandmarkerResult;
close: () => void;
};
import {
FaceLandmarker,
FilesetResolver,
} from '@mediapipe/tasks-vision';
export interface BlendShapeCategory {
categoryName: string;
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
const MODEL_URL =
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
'/mediapipe/assets/face_landmarker.task';
const WASM_ROOT =
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
'/mediapipe/wasm';
interface UseFaceLandmarkerReturn {
isLoading: boolean;
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
async function init() {
try {
// Dynamic import — avoids pulling the WASM runtime into the main bundle
const vision = await import(
/* @vite-ignore */
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
);
const { FaceLandmarker, FilesetResolver } = vision as {
FaceLandmarker: {
createFromOptions: (
resolver: unknown,
options: unknown,
) => Promise<FaceLandmarker>;
};
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
};
const filesetResolver = await FilesetResolver.forVisionTasks(
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
);
const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: {
modelAssetPath: MODEL_URL,
@@ -87,24 +53,25 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
},
outputFaceBlendshapes: true,
runningMode: 'VIDEO',
numFaces: 1,
numFaces: 3,
minFaceDetectionConfidence: 0.65,
minFacePresenceConfidence: 0.6,
minTrackingConfidence: 0.6,
});
if (!cancelled) {
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} else {
if (cancelled) {
landmarker.close();
return;
}
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} catch (err) {
if (!cancelled) {
console.error('[MediaPipe] Initialization failed:', err);
setError(
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
);
setIsLoading(false);
}
if (cancelled) return;
console.error('[MediaPipe] Initialization failed:', err);
setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
setIsLoading(false);
}
}
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
if (!landmarkerRef.current || !isReady) return null;
try {
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
} catch {
return null;
}

View File

@@ -1,17 +1,10 @@
/**
* useVelocitySocket — Manages the persistent WebSocket connection to the
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
* routes incoming events to typed handlers, and buffers outgoing packets
* during disconnection for flush-on-reconnect.
*/
import { useEffect, useRef, useCallback } from 'react';
import { useStore } from '@/store/useStore';
import { WS_URL } from '@/lib/api';
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
// Event types coming from the backend
type WsEventType =
| 'WS_ASSET_OPENED'
| 'QD_UPDATED'
@@ -26,13 +19,9 @@ interface WsMessage {
}
interface UseVelocitySocketOptions {
/** Channel name to subscribe to (appended as a path segment) */
channel?: 'notifications' | 'perception';
/** Called when the socket successfully connects */
onConnect?: () => void;
/** Called when the socket disconnects */
onDisconnect?: () => void;
/** Raw message handler — bypasses the built-in notification routing */
onMessage?: (msg: WsMessage) => void;
}
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
return;
}
// Delegate to caller's raw handler if provided
onMessage?.(msg);
// Built-in notification routing
switch (msg.type) {
case 'WS_ASSET_OPENED': {
const d = msg.data as Partial<VaultOpenedEvent>;
addNotification({
type: 'velocity_link_opened',
title: '🚨 Velocity Link Opened',
title: 'Velocity Link Opened',
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
leadId: d.lead_id,
});
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if ((d.qd_score ?? 0) >= 75) {
addNotification({
type: 'qd_spike',
title: '📈 QD Score Spike',
title: 'QD Score Spike',
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
leadId: d.lead_id,
qdScore: d.qd_score,
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if (d.tags?.length) {
addNotification({
type: 'lead_tagged',
title: '🏷️ Lead Tagged',
title: 'Lead Tagged',
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
leadId: d.lead_id,
tags: d.tags,
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
ws.onclose = () => {
onDisconnect?.();
if (!isMountedRef.current) return;
// Exponential backoff: 1s, 2s, 4s … cap at 30s
if (retryCountRef.current >= 5) return;
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
retryCountRef.current += 1;
retryTimerRef.current = setTimeout(connect, delay);
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
};
}, [connect]);
/** Send a JSON-serialisable payload; buffers if the socket is not open */
const sendPacket = useCallback((payload: unknown) => {
const str = JSON.stringify(payload);
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(str);
} else {
// Buffer up to 100 packets; drop oldest overflow
pendingBufferRef.current.push(str);
if (pendingBufferRef.current.length > 100) {
pendingBufferRef.current.shift();