sagnik/Project_Velocity
2
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix/#22 (#23)

Browse Source 
Co-authored-by: Sagnik <sagnik7896@gmail.com>
Reviewed-on: #23
This commit was merged in pull request #23.
This commit is contained in:
sayan
2026-04-13 02:35:24 +05:30
parent bf950bc789
commit 857e0b88e6
13 changed files with 26773 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
app/src/components/modules/sentinel/PerceptionPlayer.tsx
View File

@@ -41,7 +41,8 @@ import { useVelocitySocket } from '@/hooks/useVelocitySocket';
import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder';
import { API_URL } from '@/lib/api';
import { cn } from '@/lib/utils';
import type { QDScoreUpdate } from '@/types';
import { useStore } from '@/store/useStore';
import type { Lead, QDScoreUpdate } from '@/types';
// ── Types ─────────────────────────────────────────────────────────────────────
@@ -57,6 +58,13 @@ interface PerceptionPlayerProps {
const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const;
type Speed = (typeof SPEEDS)[number];
const PERCEPTION_CAMERA_CONSTRAINTS: MediaTrackConstraints = {
width: { ideal: 1280, min: 640 },
height: { ideal: 720, min: 480 },
frameRate: { ideal: 30, min: 24 },
facingMode: 'user',
};
// ── Helpers ───────────────────────────────────────────────────────────────────
function fmt(s: number): string {
@@ -194,6 +202,146 @@ function VolumeControl({
// ── Main Component ────────────────────────────────────────────────────────────
function clampScore(value: number): number {
return Math.max(1, Math.min(100, Math.round(value)));
}
function buildScoreMap(categories: Array<{ categoryName: string; score: number }>): Record<string, number> {
const map: Record<string, number> = {};
for (const category of categories) {
map[category.categoryName] = category.score;
}
return map;
}
function deriveInitialSessionScore(lead?: Lead | null, priorInteractionCount = 0): number {
const baseScore = lead?.quantumDynamicsScore ?? 50;
let modifier = 0;
const budget = (lead?.budget ?? '').toLowerCase();
const tags = (lead?.tags ?? []).map((tag) => tag.toLowerCase());
if (/(10m|15m|20m|crore|million)/.test(budget)) {
modifier += 15;
} else if (/(5m|8m)/.test(budget)) {
modifier += 8;
}
if (tags.includes('hni')) modifier += 12;
if (tags.includes('nri')) modifier += 5;
if (baseScore > 75) {
modifier += 8;
} else if (baseScore > 50) {
modifier += 4;
}
if (priorInteractionCount > 5) {
modifier += 8;
} else if (priorInteractionCount >= 2) {
modifier += 4;
}
return clampScore(baseScore + modifier);
}
function calculateLocalQdScore(previousScore: number, scoreMap: Record<string, number>): number {
const smileLeft = scoreMap.mouthSmileLeft ?? 0;
const smileRight = scoreMap.mouthSmileRight ?? 0;
const smileMax = Math.max(smileLeft, smileRight);
const browInnerUp = scoreMap.browInnerUp ?? 0;
const eyeWideLeft = scoreMap.eyeWideLeft ?? 0;
const eyeWideRight = scoreMap.eyeWideRight ?? 0;
const eyeWideMax = Math.max(eyeWideLeft, eyeWideRight);
const jawOpen = scoreMap.jawOpen ?? 0;
const cheekPuff = scoreMap.cheekPuff ?? 0;
const browDownLeft = scoreMap.browDownLeft ?? 0;
const browDownRight = scoreMap.browDownRight ?? 0;
const eyeBlinkLeft = scoreMap.eyeBlinkLeft ?? 0;
const eyeBlinkRight = scoreMap.eyeBlinkRight ?? 0;
const mouthFrown = Math.max(scoreMap.mouthFrownLeft ?? 0, scoreMap.mouthFrownRight ?? 0);
let positiveDelta = 0;
if (smileLeft > 0.5 && smileRight > 0.5) positiveDelta += 15;
else if (smileLeft > 0.5 || smileRight > 0.5) positiveDelta += 10;
if (browInnerUp > 0.4) positiveDelta += 8;
if (eyeWideMax > 0.5) positiveDelta += 7;
if (jawOpen > 0.3 && eyeWideMax > 0.5) positiveDelta += 5;
if (cheekPuff > 0.3) positiveDelta += 3;
let negativeDelta = 0;
if (browDownLeft > 0.45 && browDownRight > 0.45 && smileMax < 0.2) negativeDelta -= 10;
if (eyeBlinkLeft > 0.7 && eyeBlinkRight > 0.7 && eyeWideMax < 0.2) negativeDelta -= 15;
if (mouthFrown > 0.4) negativeDelta -= 8;
const weightedShapes = [
smileLeft,
smileRight,
browInnerUp,
eyeWideLeft,
eyeWideRight,
jawOpen,
cheekPuff,
browDownLeft,
browDownRight,
eyeBlinkLeft,
eyeBlinkRight,
mouthFrown,
];
if (weightedShapes.every((value) => value < 0.15)) negativeDelta -= 3;
let delta = positiveDelta + negativeDelta;
if (positiveDelta > 0 && negativeDelta < 0) {
delta = Math.abs(positiveDelta) >= Math.abs(negativeDelta) ? positiveDelta : negativeDelta;
}
const boundedDelta = Math.max(-20, Math.min(20, delta));
return clampScore(previousScore + boundedDelta);
}
function getFaceBounds(
landmarks: Array<{ x: number; y: number; z: number }>,
containerWidth: number,
containerHeight: number,
sourceWidth: number,
sourceHeight: number,
mirrored = true,
) {
const safeSourceWidth = sourceWidth > 0 ? sourceWidth : containerWidth;
const safeSourceHeight = sourceHeight > 0 ? sourceHeight : containerHeight;
const scale = Math.max(containerWidth / safeSourceWidth, containerHeight / safeSourceHeight);
const renderedWidth = safeSourceWidth * scale;
const renderedHeight = safeSourceHeight * scale;
const offsetX = (containerWidth - renderedWidth) / 2;
const offsetY = (containerHeight - renderedHeight) / 2;
let minX = Infinity;
let minY = Infinity;
let maxX = -Infinity;
let maxY = -Infinity;
for (const point of landmarks) {
const rawX = offsetX + (point.x * renderedWidth);
const rawY = offsetY + (point.y * renderedHeight);
const x = mirrored ? containerWidth - rawX : rawX;
const y = rawY;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
const paddingX = Math.max(8, (maxX - minX) * 0.14);
const paddingY = Math.max(8, (maxY - minY) * 0.18);
return {
x: Math.max(0, minX - paddingX),
y: Math.max(0, minY - paddingY),
width: Math.min(containerWidth, (maxX - minX) + (paddingX * 2)),
height: Math.min(containerHeight, (maxY - minY) + (paddingY * 2)),
area: Math.max(0, (maxX - minX) * (maxY - minY)),
};
}
export function PerceptionPlayer({
videoUrl, videoAssetId, leadId,
sessionMode = 'assigned', videoTitle, onSessionComplete,
@@ -202,6 +350,9 @@ export function PerceptionPlayer({
const [sessionId] = useState(
() => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`,
);
const lead = useStore((state) => state.leads.find((candidate) => candidate.id === leadId) ?? null);
const priorInteractionCount = useStore((state) => (leadId ? state.messages[leadId]?.length ?? 0 : 0));
const initialSessionScore = deriveInitialSessionScore(lead, priorInteractionCount);
// Refs
const videoRef = useRef<HTMLVideoElement>(null);
@@ -213,13 +364,15 @@ export function PerceptionPlayer({
const streamRef = useRef<MediaStream | null>(null);
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const scrubbing = useRef(false);
const qdScoreRef = useRef(initialSessionScore);
const lastBackendQdTsRef = useRef<number>(0);
// State — session
const [consentGiven, setConsentGiven] = useState(false);
const [cameraActive, setCameraActive] = useState(false);
const [cameraError, setCameraError] = useState<string | null>(null);
const [sessionActive, setSessionActive] = useState(false);
const [qdScore, setQdScore] = useState(50);
const [qdScore, setQdScore] = useState(initialSessionScore);
const [showCamera, setShowCamera] = useState(true);
// State — playback
@@ -246,10 +399,25 @@ export function PerceptionPlayer({
if (msg.type !== 'QD_UPDATED') return;
const d = msg.data as Partial<QDScoreUpdate>;
const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId;
if (match && typeof d.qd_score === 'number') setQdScore(d.qd_score);
if (match && typeof d.qd_score === 'number') {
lastBackendQdTsRef.current = Date.now();
qdScoreRef.current = d.qd_score;
setQdScore(d.qd_score);
}
}, [leadId, sessionId]),
});
useEffect(() => {
qdScoreRef.current = initialSessionScore;
setQdScore(initialSessionScore);
}, [initialSessionScore, leadId, sessionId]);
useEffect(() => {
if (!cameraActive || !streamRef.current || !webcamRef.current) return;
webcamRef.current.srcObject = streamRef.current;
void webcamRef.current.play().catch(() => null);
}, [cameraActive]);
// ── Auto-hide (YouTube behaviour) ─────────────────────────────────────────
// Show controls, reset the hide timer. After 2.8 s of no movement while
// playing, hide controls and the cursor.
@@ -289,14 +457,10 @@ export function PerceptionPlayer({
const startCamera = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' },
video: PERCEPTION_CAMERA_CONSTRAINTS,
audio: false,
});
streamRef.current = stream;
if (webcamRef.current) {
webcamRef.current.srcObject = stream;
await webcamRef.current.play();
}
setCameraActive(true);
setCameraError(null);
} catch (err) {
@@ -325,39 +489,56 @@ export function PerceptionPlayer({
const videoTsMs = Math.round(video.currentTime * 1000);
const result = detectFrame(webcam, now);
if (!result?.faceBlendshapes?.length) return;
if (!result?.faceLandmarks?.length) return;
const categories = result.faceBlendshapes[0]?.categories ?? [];
const landmarkSets = result.faceLandmarks ?? [];
const overlayWidth = canvasRef.current?.width ?? 112;
const overlayHeight = canvasRef.current?.height ?? 80;
const sourceWidth = webcam.videoWidth || overlayWidth;
const sourceHeight = webcam.videoHeight || overlayHeight;
const boundsByFace = landmarkSets.map((landmarks) =>
getFaceBounds(landmarks, overlayWidth, overlayHeight, sourceWidth, sourceHeight, true),
);
const primaryFaceIndex = boundsByFace.reduce((bestIndex, bounds, index, allBounds) => (
bounds.area > allBounds[bestIndex].area ? index : bestIndex
), 0);
const categories = result.faceBlendshapes?.[primaryFaceIndex]?.categories ?? [];
const localScoreMap = buildScoreMap(categories);
const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId);
if (!hasSignificantActivity(packet)) return;
const significantActivity = hasSignificantActivity(packet);
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
if (significantActivity) {
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
}
if (Date.now() - lastBackendQdTsRef.current > 2500) {
const rawScore = calculateLocalQdScore(qdScoreRef.current, localScoreMap);
const smoothedScore = clampScore((qdScoreRef.current * 0.7) + (rawScore * 0.3));
qdScoreRef.current = smoothedScore;
setQdScore(smoothedScore);
}
const canvas = canvasRef.current;
if (canvas && result.faceLandmarks?.length) {
if (canvas && landmarkSets.length) {
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
const lm = result.faceLandmarks[0];
if (lm?.length) {
const pts = [10, 234, 454, 152, 58, 288];
ctx.strokeStyle = 'rgba(59,130,246,0.7)';
ctx.lineWidth = 1.5;
boundsByFace.forEach((bounds, index) => {
const isPrimary = index === primaryFaceIndex;
ctx.strokeStyle = isPrimary ? 'rgba(59,130,246,0.92)' : 'rgba(255,255,255,0.5)';
ctx.lineWidth = isPrimary ? 2 : 1.2;
ctx.fillStyle = isPrimary ? 'rgba(59,130,246,0.12)' : 'rgba(255,255,255,0.06)';
ctx.beginPath();
pts.forEach((idx, i) => {
const p = lm[idx];
if (!p) return;
i === 0 ? ctx.moveTo(p.x * canvas.width, p.y * canvas.height)
: ctx.lineTo(p.x * canvas.width, p.y * canvas.height);
});
ctx.closePath();
ctx.roundRect(bounds.x, bounds.y, bounds.width, bounds.height, 8);
ctx.fill();
ctx.stroke();
}
});
}
}
}, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]);

1
app/src/global.d.ts vendored
View File

@@ -1 +0,0 @@
declare module 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs';

83
app/src/hooks/useMediapipeFaceLandmarker.ts
View File

@@ -1,27 +1,8 @@
/**
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
* FaceLandmarker WASM task and exposes a frame-processing callback.
*
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
* CDN path). Initialization is async; `isReady` is false until complete.
*
* Usage:
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
* // In rAF loop:
* const result = detectFrame(videoElement, performance.now());
*/
import { useState, useEffect, useRef, useCallback } from 'react';
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues.
// Types are inlined below so we don't need @types/mediapipe
type FaceLandmarker = {
detectForVideo: (
videoElement: HTMLVideoElement,
timestamp: number,
) => FaceLandmarkerResult;
close: () => void;
};
import {
FaceLandmarker,
FilesetResolver,
} from '@mediapipe/tasks-vision';
export interface BlendShapeCategory {
categoryName: string;
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
const MODEL_URL =
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
'/mediapipe/assets/face_landmarker.task';
const WASM_ROOT =
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
'/mediapipe/wasm';
interface UseFaceLandmarkerReturn {
isLoading: boolean;
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
async function init() {
try {
// Dynamic import — avoids pulling the WASM runtime into the main bundle
const vision = await import(
/* @vite-ignore */
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
);
const { FaceLandmarker, FilesetResolver } = vision as {
FaceLandmarker: {
createFromOptions: (
resolver: unknown,
options: unknown,
) => Promise<FaceLandmarker>;
};
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
};
const filesetResolver = await FilesetResolver.forVisionTasks(
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
);
const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: {
modelAssetPath: MODEL_URL,
@@ -87,24 +53,25 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
},
outputFaceBlendshapes: true,
runningMode: 'VIDEO',
numFaces: 1,
numFaces: 3,
minFaceDetectionConfidence: 0.65,
minFacePresenceConfidence: 0.6,
minTrackingConfidence: 0.6,
});
if (!cancelled) {
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} else {
if (cancelled) {
landmarker.close();
return;
}
landmarkerRef.current = landmarker;
setIsLoading(false);
setIsReady(true);
} catch (err) {
if (!cancelled) {
console.error('[MediaPipe] Initialization failed:', err);
setError(
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
);
setIsLoading(false);
}
if (cancelled) return;
console.error('[MediaPipe] Initialization failed:', err);
setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
setIsLoading(false);
}
}
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
if (!landmarkerRef.current || !isReady) return null;
try {
return landmarkerRef.current.detectForVideo(videoElement, timestampMs);
return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
} catch {
return null;
}

25
app/src/hooks/useVelocitySocket.ts
View File

@@ -1,17 +1,10 @@
/**
* useVelocitySocket — Manages the persistent WebSocket connection to the
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
* routes incoming events to typed handlers, and buffers outgoing packets
* during disconnection for flush-on-reconnect.
*/
import { useEffect, useRef, useCallback } from 'react';
import { useStore } from '@/store/useStore';
import { WS_URL } from '@/lib/api';
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
// Event types coming from the backend
type WsEventType =
| 'WS_ASSET_OPENED'
| 'QD_UPDATED'
@@ -26,13 +19,9 @@ interface WsMessage {
}
interface UseVelocitySocketOptions {
/** Channel name to subscribe to (appended as a path segment) */
channel?: 'notifications' | 'perception';
/** Called when the socket successfully connects */
onConnect?: () => void;
/** Called when the socket disconnects */
onDisconnect?: () => void;
/** Raw message handler — bypasses the built-in notification routing */
onMessage?: (msg: WsMessage) => void;
}
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
return;
}
// Delegate to caller's raw handler if provided
onMessage?.(msg);
// Built-in notification routing
switch (msg.type) {
case 'WS_ASSET_OPENED': {
const d = msg.data as Partial<VaultOpenedEvent>;
addNotification({
type: 'velocity_link_opened',
title: '🚨 Velocity Link Opened',
title: 'Velocity Link Opened',
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
leadId: d.lead_id,
});
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if ((d.qd_score ?? 0) >= 75) {
addNotification({
type: 'qd_spike',
title: '📈 QD Score Spike',
title: 'QD Score Spike',
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
leadId: d.lead_id,
qdScore: d.qd_score,
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if (d.tags?.length) {
addNotification({
type: 'lead_tagged',
title: '🏷️ Lead Tagged',
title: 'Lead Tagged',
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
leadId: d.lead_id,
tags: d.tags,
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
ws.onclose = () => {
onDisconnect?.();
if (!isMountedRef.current) return;
// Exponential backoff: 1s, 2s, 4s … cap at 30s
if (retryCountRef.current >= 5) return;
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
retryCountRef.current += 1;
retryTimerRef.current = setTimeout(connect, delay);
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
};
}, [connect]);
/** Send a JSON-serialisable payload; buffers if the socket is not open */
const sendPacket = useCallback((payload: unknown) => {
const str = JSON.stringify(payload);
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(str);
} else {
// Buffer up to 100 packets; drop oldest overflow
pendingBufferRef.current.push(str);
if (pendingBufferRef.current.length > 100) {
pendingBufferRef.current.shift();