sagnik/Project_Velocity
2
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix/#22 #23

Merged
sayan merged 3 commits from fix/#22 into main 2026-04-13 02:35:28 +05:30
Conversation 0 Commits 3 Files Changed 22 +27053 -121
22 changed files with 27053 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/dist/index.html vendored
View File

@@ -4,7 +4,7 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Velocity WebOS</title> <title>Velocity WebOS</title>
<script type="module" crossorigin src="./assets/index-CJRJmEe7.js"></script> <script type="module" crossorigin src="./assets/index-BYTPd1oW.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-UA0RXBVG.css"> <link rel="stylesheet" crossorigin href="./assets/index-UA0RXBVG.css">
</head> </head>
<body> <body>

13
app/package-lock.json generated
View File

@@ -12,6 +12,7 @@
"@dnd-kit/sortable": "^10.0.0", "@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2", "@dnd-kit/utilities": "^3.2.2",
"@hookform/resolvers": "^5.2.2", "@hookform/resolvers": "^5.2.2",
"@mediapipe/tasks-vision": "^0.10.34",
"@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.8", "@radix-ui/react-aspect-ratio": "^1.1.8",
@@ -2668,9 +2669,9 @@
} }
}, },
"node_modules/@mediapipe/tasks-vision": { "node_modules/@mediapipe/tasks-vision": {
"version": "0.10.17", "version": "0.10.34",
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz", "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.34.tgz",
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==", "integrity": "sha512-KFGyhDsjJ+9WUMcMfjTOpcEp3LJNS3KwC7BfvKrCYELn/7G/5kmwnU7z6Spps+iWQoTGL8xW8i68r65OTa3DwA==",
"license": "Apache-2.0" "license": "Apache-2.0"
}, },
"node_modules/@monogrid/gainmap-js": { "node_modules/@monogrid/gainmap-js": {
@@ -4388,6 +4389,12 @@
} }
} }
}, },
"node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": {
"version": "0.10.17",
"resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
"integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
"license": "Apache-2.0"
},
"node_modules/@react-three/fiber": { "node_modules/@react-three/fiber": {
"version": "9.5.0", "version": "9.5.0",
"resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.5.0.tgz", "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.5.0.tgz",

3
app/package.json
View File

@@ -15,6 +15,7 @@
"@dnd-kit/sortable": "^10.0.0", "@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2", "@dnd-kit/utilities": "^3.2.2",
"@hookform/resolvers": "^5.2.2", "@hookform/resolvers": "^5.2.2",
"@mediapipe/tasks-vision": "^0.10.34",
"@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.8", "@radix-ui/react-aspect-ratio": "^1.1.8",
@@ -67,8 +68,8 @@
"zustand": "^5.0.11" "zustand": "^5.0.11"
}, },
"devDependencies": { "devDependencies": {
"@playwright/test": "^1.55.0",
"@eslint/js": "^9.39.1", "@eslint/js": "^9.39.1",
"@playwright/test": "^1.55.0",
"@types/node": "^24.10.1", "@types/node": "^24.10.1",
"@types/react": "^19.2.5", "@types/react": "^19.2.5",
"@types/react-dom": "^19.2.3", "@types/react-dom": "^19.2.3",

BIN
app/public/mediapipe/assets/face_landmarker.task Normal file
View File

Binary file not shown.

8844
app/public/mediapipe/wasm/vision_wasm_internal.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
app/public/mediapipe/wasm/vision_wasm_internal.wasm Normal file
View File

Binary file not shown.

8840
app/public/mediapipe/wasm/vision_wasm_module_internal.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
app/public/mediapipe/wasm/vision_wasm_module_internal.wasm Normal file
View File

Binary file not shown.

8835
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
app/public/mediapipe/wasm/vision_wasm_nosimd_internal.wasm Normal file
View File

Binary file not shown.

231
app/src/components/modules/sentinel/PerceptionPlayer.tsx
View File

@@ -41,7 +41,8 @@ import { useVelocitySocket } from '@/hooks/useVelocitySocket';
import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder'; import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder';
import { API_URL } from '@/lib/api'; import { API_URL } from '@/lib/api';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import type { QDScoreUpdate } from '@/types'; import { useStore } from '@/store/useStore';
import type { Lead, QDScoreUpdate } from '@/types';
// ── Types ───────────────────────────────────────────────────────────────────── // ── Types ─────────────────────────────────────────────────────────────────────
@@ -57,6 +58,13 @@ interface PerceptionPlayerProps {
const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const; const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const;
type Speed = (typeof SPEEDS)[number]; type Speed = (typeof SPEEDS)[number];
const PERCEPTION_CAMERA_CONSTRAINTS: MediaTrackConstraints = {
width: { ideal: 1280, min: 640 },
height: { ideal: 720, min: 480 },
frameRate: { ideal: 30, min: 24 },
facingMode: 'user',
};
// ── Helpers ─────────────────────────────────────────────────────────────────── // ── Helpers ───────────────────────────────────────────────────────────────────
function fmt(s: number): string { function fmt(s: number): string {
@@ -194,6 +202,146 @@ function VolumeControl({
// ── Main Component ──────────────────────────────────────────────────────────── // ── Main Component ────────────────────────────────────────────────────────────
function clampScore(value: number): number {
return Math.max(1, Math.min(100, Math.round(value)));
}
function buildScoreMap(categories: Array<{ categoryName: string; score: number }>): Record<string, number> {
const map: Record<string, number> = {};
for (const category of categories) {
map[category.categoryName] = category.score;
}
return map;
}
function deriveInitialSessionScore(lead?: Lead | null, priorInteractionCount = 0): number {
const baseScore = lead?.quantumDynamicsScore ?? 50;
let modifier = 0;
const budget = (lead?.budget ?? '').toLowerCase();
const tags = (lead?.tags ?? []).map((tag) => tag.toLowerCase());
if (/(10m|15m|20m|crore|million)/.test(budget)) {
modifier += 15;
} else if (/(5m|8m)/.test(budget)) {
modifier += 8;
}
if (tags.includes('hni')) modifier += 12;
if (tags.includes('nri')) modifier += 5;
if (baseScore > 75) {
modifier += 8;
} else if (baseScore > 50) {
modifier += 4;
}
if (priorInteractionCount > 5) {
modifier += 8;
} else if (priorInteractionCount >= 2) {
modifier += 4;
}
return clampScore(baseScore + modifier);
}
function calculateLocalQdScore(previousScore: number, scoreMap: Record<string, number>): number {
const smileLeft = scoreMap.mouthSmileLeft ?? 0;
const smileRight = scoreMap.mouthSmileRight ?? 0;
const smileMax = Math.max(smileLeft, smileRight);
const browInnerUp = scoreMap.browInnerUp ?? 0;
const eyeWideLeft = scoreMap.eyeWideLeft ?? 0;
const eyeWideRight = scoreMap.eyeWideRight ?? 0;
const eyeWideMax = Math.max(eyeWideLeft, eyeWideRight);
const jawOpen = scoreMap.jawOpen ?? 0;
const cheekPuff = scoreMap.cheekPuff ?? 0;
const browDownLeft = scoreMap.browDownLeft ?? 0;
const browDownRight = scoreMap.browDownRight ?? 0;
const eyeBlinkLeft = scoreMap.eyeBlinkLeft ?? 0;
const eyeBlinkRight = scoreMap.eyeBlinkRight ?? 0;
const mouthFrown = Math.max(scoreMap.mouthFrownLeft ?? 0, scoreMap.mouthFrownRight ?? 0);
let positiveDelta = 0;
if (smileLeft > 0.5 && smileRight > 0.5) positiveDelta += 15;
else if (smileLeft > 0.5 || smileRight > 0.5) positiveDelta += 10;
if (browInnerUp > 0.4) positiveDelta += 8;
if (eyeWideMax > 0.5) positiveDelta += 7;
if (jawOpen > 0.3 && eyeWideMax > 0.5) positiveDelta += 5;
if (cheekPuff > 0.3) positiveDelta += 3;
let negativeDelta = 0;
if (browDownLeft > 0.45 && browDownRight > 0.45 && smileMax < 0.2) negativeDelta -= 10;
if (eyeBlinkLeft > 0.7 && eyeBlinkRight > 0.7 && eyeWideMax < 0.2) negativeDelta -= 15;
if (mouthFrown > 0.4) negativeDelta -= 8;
const weightedShapes = [
smileLeft,
smileRight,
browInnerUp,
eyeWideLeft,
eyeWideRight,
jawOpen,
cheekPuff,
browDownLeft,
browDownRight,
eyeBlinkLeft,
eyeBlinkRight,
mouthFrown,
];
if (weightedShapes.every((value) => value < 0.15)) negativeDelta -= 3;
let delta = positiveDelta + negativeDelta;
if (positiveDelta > 0 && negativeDelta < 0) {
delta = Math.abs(positiveDelta) >= Math.abs(negativeDelta) ? positiveDelta : negativeDelta;
}
const boundedDelta = Math.max(-20, Math.min(20, delta));
return clampScore(previousScore + boundedDelta);
}
function getFaceBounds(
landmarks: Array<{ x: number; y: number; z: number }>,
containerWidth: number,
containerHeight: number,
sourceWidth: number,
sourceHeight: number,
mirrored = true,
) {
const safeSourceWidth = sourceWidth > 0 ? sourceWidth : containerWidth;
const safeSourceHeight = sourceHeight > 0 ? sourceHeight : containerHeight;
const scale = Math.max(containerWidth / safeSourceWidth, containerHeight / safeSourceHeight);
const renderedWidth = safeSourceWidth * scale;
const renderedHeight = safeSourceHeight * scale;
const offsetX = (containerWidth - renderedWidth) / 2;
const offsetY = (containerHeight - renderedHeight) / 2;
let minX = Infinity;
let minY = Infinity;
let maxX = -Infinity;
let maxY = -Infinity;
for (const point of landmarks) {
const rawX = offsetX + (point.x * renderedWidth);
const rawY = offsetY + (point.y * renderedHeight);
const x = mirrored ? containerWidth - rawX : rawX;
const y = rawY;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
const paddingX = Math.max(8, (maxX - minX) * 0.14);
const paddingY = Math.max(8, (maxY - minY) * 0.18);
return {
x: Math.max(0, minX - paddingX),
y: Math.max(0, minY - paddingY),
width: Math.min(containerWidth, (maxX - minX) + (paddingX * 2)),
height: Math.min(containerHeight, (maxY - minY) + (paddingY * 2)),
area: Math.max(0, (maxX - minX) * (maxY - minY)),
};
}
export function PerceptionPlayer({ export function PerceptionPlayer({
videoUrl, videoAssetId, leadId, videoUrl, videoAssetId, leadId,
sessionMode = 'assigned', videoTitle, onSessionComplete, sessionMode = 'assigned', videoTitle, onSessionComplete,
@@ -202,6 +350,9 @@ export function PerceptionPlayer({
const [sessionId] = useState( const [sessionId] = useState(
() => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`, () => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`,
); );
const lead = useStore((state) => state.leads.find((candidate) => candidate.id === leadId) ?? null);
const priorInteractionCount = useStore((state) => (leadId ? state.messages[leadId]?.length ?? 0 : 0));
const initialSessionScore = deriveInitialSessionScore(lead, priorInteractionCount);
// Refs // Refs
const videoRef = useRef<HTMLVideoElement>(null); const videoRef = useRef<HTMLVideoElement>(null);
@@ -213,13 +364,15 @@ export function PerceptionPlayer({
const streamRef = useRef<MediaStream | null>(null); const streamRef = useRef<MediaStream | null>(null);
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null); const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const scrubbing = useRef(false); const scrubbing = useRef(false);
const qdScoreRef = useRef(initialSessionScore);
const lastBackendQdTsRef = useRef<number>(0);
// State — session // State — session
const [consentGiven, setConsentGiven] = useState(false); const [consentGiven, setConsentGiven] = useState(false);
const [cameraActive, setCameraActive] = useState(false); const [cameraActive, setCameraActive] = useState(false);
const [cameraError, setCameraError] = useState<string | null>(null); const [cameraError, setCameraError] = useState<string | null>(null);
const [sessionActive, setSessionActive] = useState(false); const [sessionActive, setSessionActive] = useState(false);
const [qdScore, setQdScore] = useState(50); const [qdScore, setQdScore] = useState(initialSessionScore);
const [showCamera, setShowCamera] = useState(true); const [showCamera, setShowCamera] = useState(true);
// State — playback // State — playback
@@ -246,10 +399,25 @@ export function PerceptionPlayer({
if (msg.type !== 'QD_UPDATED') return; if (msg.type !== 'QD_UPDATED') return;
const d = msg.data as Partial<QDScoreUpdate>; const d = msg.data as Partial<QDScoreUpdate>;
const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId; const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId;
if (match && typeof d.qd_score === 'number') setQdScore(d.qd_score); if (match && typeof d.qd_score === 'number') {
lastBackendQdTsRef.current = Date.now();
qdScoreRef.current = d.qd_score;
setQdScore(d.qd_score);
}
}, [leadId, sessionId]), }, [leadId, sessionId]),
}); });
useEffect(() => {
qdScoreRef.current = initialSessionScore;
setQdScore(initialSessionScore);
}, [initialSessionScore, leadId, sessionId]);
useEffect(() => {
if (!cameraActive || !streamRef.current || !webcamRef.current) return;
webcamRef.current.srcObject = streamRef.current;
void webcamRef.current.play().catch(() => null);
}, [cameraActive]);
// ── Auto-hide (YouTube behaviour) ───────────────────────────────────────── // ── Auto-hide (YouTube behaviour) ─────────────────────────────────────────
// Show controls, reset the hide timer. After 2.8 s of no movement while // Show controls, reset the hide timer. After 2.8 s of no movement while
// playing, hide controls and the cursor. // playing, hide controls and the cursor.
@@ -289,14 +457,10 @@ export function PerceptionPlayer({
const startCamera = useCallback(async () => { const startCamera = useCallback(async () => {
try { try {
const stream = await navigator.mediaDevices.getUserMedia({ const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' }, video: PERCEPTION_CAMERA_CONSTRAINTS,
audio: false, audio: false,
}); });
streamRef.current = stream; streamRef.current = stream;
if (webcamRef.current) {
webcamRef.current.srcObject = stream;
await webcamRef.current.play();
}
setCameraActive(true); setCameraActive(true);
setCameraError(null); setCameraError(null);
} catch (err) { } catch (err) {
@@ -325,39 +489,56 @@ export function PerceptionPlayer({
const videoTsMs = Math.round(video.currentTime * 1000); const videoTsMs = Math.round(video.currentTime * 1000);
const result = detectFrame(webcam, now); const result = detectFrame(webcam, now);
if (!result?.faceBlendshapes?.length) return; if (!result?.faceLandmarks?.length) return;
const categories = result.faceBlendshapes[0]?.categories ?? []; const landmarkSets = result.faceLandmarks ?? [];
const overlayWidth = canvasRef.current?.width ?? 112;
const overlayHeight = canvasRef.current?.height ?? 80;
const sourceWidth = webcam.videoWidth || overlayWidth;
const sourceHeight = webcam.videoHeight || overlayHeight;
const boundsByFace = landmarkSets.map((landmarks) =>
getFaceBounds(landmarks, overlayWidth, overlayHeight, sourceWidth, sourceHeight, true),
);
const primaryFaceIndex = boundsByFace.reduce((bestIndex, bounds, index, allBounds) => (
bounds.area > allBounds[bestIndex].area ? index : bestIndex
), 0);
const categories = result.faceBlendshapes?.[primaryFaceIndex]?.categories ?? [];
const localScoreMap = buildScoreMap(categories);
const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId); const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId);
if (!hasSignificantActivity(packet)) return; const significantActivity = hasSignificantActivity(packet);
if (significantActivity) {
sendPacket({ sendPacket({
event: 'BIOMETRIC_PACKET', event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId, lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId, session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes, video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
}); });
}
if (Date.now() - lastBackendQdTsRef.current > 2500) {
const rawScore = calculateLocalQdScore(qdScoreRef.current, localScoreMap);
const smoothedScore = clampScore((qdScoreRef.current * 0.7) + (rawScore * 0.3));
qdScoreRef.current = smoothedScore;
setQdScore(smoothedScore);
}
const canvas = canvasRef.current; const canvas = canvasRef.current;
if (canvas && result.faceLandmarks?.length) { if (canvas && landmarkSets.length) {
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
if (ctx) { if (ctx) {
ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.clearRect(0, 0, canvas.width, canvas.height);
const lm = result.faceLandmarks[0]; boundsByFace.forEach((bounds, index) => {
if (lm?.length) { const isPrimary = index === primaryFaceIndex;
const pts = [10, 234, 454, 152, 58, 288]; ctx.strokeStyle = isPrimary ? 'rgba(59,130,246,0.92)' : 'rgba(255,255,255,0.5)';
ctx.strokeStyle = 'rgba(59,130,246,0.7)'; ctx.lineWidth = isPrimary ? 2 : 1.2;
ctx.lineWidth = 1.5; ctx.fillStyle = isPrimary ? 'rgba(59,130,246,0.12)' : 'rgba(255,255,255,0.06)';
ctx.beginPath(); ctx.beginPath();
pts.forEach((idx, i) => { ctx.roundRect(bounds.x, bounds.y, bounds.width, bounds.height, 8);
const p = lm[idx]; ctx.fill();
if (!p) return;
i === 0 ? ctx.moveTo(p.x * canvas.width, p.y * canvas.height)
: ctx.lineTo(p.x * canvas.width, p.y * canvas.height);
});
ctx.closePath();
ctx.stroke(); ctx.stroke();
} });
} }
} }
}, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]); }, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]);

1
app/src/global.d.ts vendored
View File

@@ -1 +0,0 @@
declare module 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs';

77
app/src/hooks/useMediapipeFaceLandmarker.ts
View File

@@ -1,27 +1,8 @@
/**
* useMediapipeFaceLandmarker — Custom hook that initialises the MediaPipe
* FaceLandmarker WASM task and exposes a frame-processing callback.
*
* The model file must be hosted at VITE_MEDIAPIPE_MODEL_URL (or the default
* CDN path). Initialization is async; `isReady` is false until complete.
*
* Usage:
* const { isReady, isLoading, detectFrame, error } = useMediapipeFaceLandmarker();
* // In rAF loop:
* const result = detectFrame(videoElement, performance.now());
*/
import { useState, useEffect, useRef, useCallback } from 'react'; import { useState, useEffect, useRef, useCallback } from 'react';
import {
// MediaPipe tasks-vision is loaded dynamically to avoid SSR issues. FaceLandmarker,
// Types are inlined below so we don't need @types/mediapipe FilesetResolver,
type FaceLandmarker = { } from '@mediapipe/tasks-vision';
detectForVideo: (
videoElement: HTMLVideoElement,
timestamp: number,
) => FaceLandmarkerResult;
close: () => void;
};
export interface BlendShapeCategory { export interface BlendShapeCategory {
categoryName: string; categoryName: string;
@@ -37,7 +18,11 @@ export interface FaceLandmarkerResult {
const MODEL_URL = const MODEL_URL =
import.meta.env.VITE_MEDIAPIPE_MODEL_URL ?? import.meta.env.VITE_MEDIAPIPE_MODEL_URL ??
'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task'; '/mediapipe/assets/face_landmarker.task';
const WASM_ROOT =
import.meta.env.VITE_MEDIAPIPE_WASM_ROOT ??
'/mediapipe/wasm';
interface UseFaceLandmarkerReturn { interface UseFaceLandmarkerReturn {
isLoading: boolean; isLoading: boolean;
@@ -60,26 +45,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
async function init() { async function init() {
try { try {
// Dynamic import — avoids pulling the WASM runtime into the main bundle const filesetResolver = await FilesetResolver.forVisionTasks(WASM_ROOT);
const vision = await import(
/* @vite-ignore */
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/vision_bundle.mjs'
);
const { FaceLandmarker, FilesetResolver } = vision as {
FaceLandmarker: {
createFromOptions: (
resolver: unknown,
options: unknown,
) => Promise<FaceLandmarker>;
};
FilesetResolver: { forVisionTasks: (path: string) => Promise<unknown> };
};
const filesetResolver = await FilesetResolver.forVisionTasks(
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22/wasm',
);
const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, { const landmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: { baseOptions: {
modelAssetPath: MODEL_URL, modelAssetPath: MODEL_URL,
@@ -87,26 +53,27 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
}, },
outputFaceBlendshapes: true, outputFaceBlendshapes: true,
runningMode: 'VIDEO', runningMode: 'VIDEO',
numFaces: 1, numFaces: 3,
minFaceDetectionConfidence: 0.65,
minFacePresenceConfidence: 0.6,
minTrackingConfidence: 0.6,
}); });
if (!cancelled) { if (cancelled) {
landmarker.close();
return;
}
landmarkerRef.current = landmarker; landmarkerRef.current = landmarker;
setIsLoading(false); setIsLoading(false);
setIsReady(true); setIsReady(true);
} else {
landmarker.close();
}
} catch (err) { } catch (err) {
if (!cancelled) { if (cancelled) return;
console.error('[MediaPipe] Initialization failed:', err); console.error('[MediaPipe] Initialization failed:', err);
setError( setError(err instanceof Error ? err.message : 'MediaPipe failed to initialize.');
err instanceof Error ? err.message : 'MediaPipe failed to initialize.',
);
setIsLoading(false); setIsLoading(false);
} }
} }
}
void init(); void init();
@@ -121,7 +88,7 @@ export function useMediapipeFaceLandmarker(): UseFaceLandmarkerReturn {
(videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => { (videoElement: HTMLVideoElement, timestampMs: number): FaceLandmarkerResult | null => {
if (!landmarkerRef.current || !isReady) return null; if (!landmarkerRef.current || !isReady) return null;
try { try {
return landmarkerRef.current.detectForVideo(videoElement, timestampMs); return landmarkerRef.current.detectForVideo(videoElement, timestampMs) as FaceLandmarkerResult;
} catch { } catch {
return null; return null;
} }

25
app/src/hooks/useVelocitySocket.ts
View File

@@ -1,17 +1,10 @@
/**
* useVelocitySocket — Manages the persistent WebSocket connection to the
* Velocity FastAPI backend. Handles reconnection with exponential backoff,
* routes incoming events to typed handlers, and buffers outgoing packets
* during disconnection for flush-on-reconnect.
*/
import { useEffect, useRef, useCallback } from 'react'; import { useEffect, useRef, useCallback } from 'react';
import { useStore } from '@/store/useStore'; import { useStore } from '@/store/useStore';
import { WS_URL } from '@/lib/api'; import { WS_URL } from '@/lib/api';
import type { QDScoreUpdate, VaultOpenedEvent } from '@/types'; import type { QDScoreUpdate, VaultOpenedEvent } from '@/types';
const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`; const SENTINEL_WS_ROOT = `${WS_URL}/api/sentinel/ws`;
// Event types coming from the backend
type WsEventType = type WsEventType =
| 'WS_ASSET_OPENED' | 'WS_ASSET_OPENED'
| 'QD_UPDATED' | 'QD_UPDATED'
@@ -26,13 +19,9 @@ interface WsMessage {
} }
interface UseVelocitySocketOptions { interface UseVelocitySocketOptions {
/** Channel name to subscribe to (appended as a path segment) */
channel?: 'notifications' | 'perception'; channel?: 'notifications' | 'perception';
/** Called when the socket successfully connects */
onConnect?: () => void; onConnect?: () => void;
/** Called when the socket disconnects */
onDisconnect?: () => void; onDisconnect?: () => void;
/** Raw message handler — bypasses the built-in notification routing */
onMessage?: (msg: WsMessage) => void; onMessage?: (msg: WsMessage) => void;
} }
@@ -55,16 +44,14 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
return; return;
} }
// Delegate to caller's raw handler if provided
onMessage?.(msg); onMessage?.(msg);
// Built-in notification routing
switch (msg.type) { switch (msg.type) {
case 'WS_ASSET_OPENED': { case 'WS_ASSET_OPENED': {
const d = msg.data as Partial<VaultOpenedEvent>; const d = msg.data as Partial<VaultOpenedEvent>;
addNotification({ addNotification({
type: 'velocity_link_opened', type: 'velocity_link_opened',
title: '🚨 Velocity Link Opened', title: 'Velocity Link Opened',
body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`, body: `${d.lead_name ?? 'A prospect'} just opened ${d.asset_name ?? 'your asset'}.`,
leadId: d.lead_id, leadId: d.lead_id,
}); });
@@ -75,7 +62,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if ((d.qd_score ?? 0) >= 75) { if ((d.qd_score ?? 0) >= 75) {
addNotification({ addNotification({
type: 'qd_spike', type: 'qd_spike',
title: '📈 QD Score Spike', title: 'QD Score Spike',
body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(), body: `QD Score jumped to ${d.qd_score}. ${d.reasoning ?? ''}`.trim(),
leadId: d.lead_id, leadId: d.lead_id,
qdScore: d.qd_score, qdScore: d.qd_score,
@@ -88,7 +75,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
if (d.tags?.length) { if (d.tags?.length) {
addNotification({ addNotification({
type: 'lead_tagged', type: 'lead_tagged',
title: '🏷️ Lead Tagged', title: 'Lead Tagged',
body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`, body: `${d.lead_name ?? 'Lead'} tagged as ${d.tags.join(', ')}.`,
leadId: d.lead_id, leadId: d.lead_id,
tags: d.tags, tags: d.tags,
@@ -122,7 +109,7 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
ws.onclose = () => { ws.onclose = () => {
onDisconnect?.(); onDisconnect?.();
if (!isMountedRef.current) return; if (!isMountedRef.current) return;
// Exponential backoff: 1s, 2s, 4s … cap at 30s if (retryCountRef.current >= 5) return;
const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000); const delay = Math.min(1000 * 2 ** retryCountRef.current, 30_000);
retryCountRef.current += 1; retryCountRef.current += 1;
retryTimerRef.current = setTimeout(connect, delay); retryTimerRef.current = setTimeout(connect, delay);
@@ -142,13 +129,11 @@ export function useVelocitySocket(options: UseVelocitySocketOptions = {}) {
}; };
}, [connect]); }, [connect]);
/** Send a JSON-serialisable payload; buffers if the socket is not open */
const sendPacket = useCallback((payload: unknown) => { const sendPacket = useCallback((payload: unknown) => {
const str = JSON.stringify(payload); const str = JSON.stringify(payload);
if (wsRef.current?.readyState === WebSocket.OPEN) { if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(str); wsRef.current.send(str);
} else { } else {
// Buffer up to 100 packets; drop oldest overflow
pendingBufferRef.current.push(str); pendingBufferRef.current.push(str);
if (pendingBufferRef.current.length > 100) { if (pendingBufferRef.current.length > 100) {
pendingBufferRef.current.shift(); pendingBufferRef.current.shift();

2
app/src/lib/api.ts
View File

@@ -1,5 +1,5 @@
const rawApiBase = import.meta.env.VITE_API_URL?.trim(); const rawApiBase = import.meta.env.VITE_API_URL?.trim();
const DEPLOYED_BACKEND_ORIGIN = 'https://54.152.236.10'; const DEPLOYED_BACKEND_ORIGIN = 'https://api.desineuron.in';
function getBrowserOrigin() { function getBrowserOrigin() {
if (typeof window !== 'undefined' && window.location?.origin) { if (typeof window !== 'undefined' && window.location?.origin) {

11
app/vite.config.ts
View File

@@ -3,10 +3,15 @@ import react from "@vitejs/plugin-react"
import { defineConfig } from "vite" import { defineConfig } from "vite"
import { inspectAttr } from 'kimi-plugin-inspect-react' import { inspectAttr } from 'kimi-plugin-inspect-react'
const backendProxyTarget = process.env.VITE_BACKEND_PROXY_TARGET?.trim() || "https://api.desineuron.in"
// https://vite.dev/config/ // https://vite.dev/config/
export default defineConfig({ export default defineConfig({
base: './', base: './',
plugins: [inspectAttr(), react()], plugins: [inspectAttr(), react()],
optimizeDeps: {
exclude: ['@mediapipe/tasks-vision'],
},
resolve: { resolve: {
alias: { alias: {
"@": path.resolve(__dirname, "./src"), "@": path.resolve(__dirname, "./src"),
@@ -17,18 +22,18 @@ export default defineConfig({
port: 5173, port: 5173,
proxy: { proxy: {
"/api": { "/api": {
target: "https://54.152.236.10", target: backendProxyTarget,
changeOrigin: true, changeOrigin: true,
secure: false, secure: false,
ws: true, ws: true,
}, },
"/assets": { "/assets": {
target: "https://54.152.236.10", target: backendProxyTarget,
changeOrigin: true, changeOrigin: true,
secure: false, secure: false,
}, },
"/vault": { "/vault": {
target: "https://54.152.236.10", target: backendProxyTarget,
changeOrigin: true, changeOrigin: true,
secure: false, secure: false,
}, },

74
infrastructure/desineuron_ingress/TEAM_HANDOFF_2026-04-08.md
View File

@@ -21,6 +21,7 @@ Date: 2026-04-08
15. Team Summary 15. Team Summary
16. Current Status Snapshot - 2026-04-12 16. Current Status Snapshot - 2026-04-12
17. Linux Ops Control Plane 17. Linux Ops Control Plane
18. Velocity Stable API Runbook
### Outcome ### Outcome
@@ -589,3 +590,76 @@ Reference docs:
- [README.md](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/ops_control_plane/README.md) - [README.md](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/ops_control_plane/README.md)
- [Desineuron Ops Control Plane Bibel.md](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/.Agent%20Context/Bibels/Desineuron%20Ops%20Control%20Plane%20Bibel.md) - [Desineuron Ops Control Plane Bibel.md](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/.Agent%20Context/Bibels/Desineuron%20Ops%20Control%20Plane%20Bibel.md)
### Velocity Stable API Runbook
Problem:
- the Velocity backend was still exposed through an ephemeral AWS instance IP
- frontend code was hardcoded to `https://54.152.236.10`
- EC2 stop/start changed the backend public IP and broke the app
- the stable ingress already existed, but Velocity had never been mapped through it
Correct production pattern:
- public API hostname: `api.desineuron.in`
- public edge: ingress `98.87.120.120`
- ingress route target: current private IP of the EC2 instance tagged `DesineuronRole=velocity-backend`
- Linux box runs the route-sync timer, just like the ComfyUI pattern
- backend stays private and should only accept `8000/8001` from ingress security group `sg-0721b8b48e12c531d`
Repo artifacts added for this pattern:
- [sync_velocity_route.py](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/desineuron_ingress/sync_velocity_route.py)
- [desineuron-velocity-route-sync.service](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/desineuron_ingress/desineuron-velocity-route-sync.service)
- [desineuron-velocity-route-sync.timer](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/desineuron_ingress/desineuron-velocity-route-sync.timer)
- [install_linux_velocity_route_sync.sh](/F:/Workin%20In%20Progress/DESINEURON/GITLAB/Project_Velocity/infrastructure/desineuron_ingress/install_linux_velocity_route_sync.sh)
Frontend changes expected by this pattern:
- `app/src/lib/api.ts` now points production traffic to `https://api.desineuron.in`
- `app/vite.config.ts` uses `VITE_BACKEND_PROXY_TARGET` for local dev override
- Vite proxy errors are no longer tied to one stale EC2 IP
Backend bootstrap note:
- `remote_bootstrap_20260401.sh` now includes:
- `https://api.desineuron.in`
- `https://54.152.236.10`
- `https://18.212.122.77`
in `CORS_ORIGINS`
Operator steps still required outside the repo:
1. Tag the backend EC2 instance:
- key: `DesineuronRole`
- value: `velocity-backend`
2. Add Cloudflare DNS:
- record: `api.desineuron.in`
- type: `A`
- value: `98.87.120.120`
- proxy: `DNS only`
3. Bootstrap the first ingress route once:
- target host: current backend private IP
- target port: `8001` unless the backend listener is changed
4. Lock down backend security group:
- revoke public `0.0.0.0/0` access to the backend app port
- allow backend app port only from ingress security group `sg-0721b8b48e12c531d`
5. Update backend runtime env and restart:
- add `https://api.desineuron.in` to `CORS_ORIGINS`
- restart `velocity-backend.service`
6. Install the Linux route sync timer:
- copy `infrastructure/desineuron_ingress/*velocity*` to Linux temporary staging
- run `install_linux_velocity_route_sync.sh`
Expected result after the 6 steps:
- frontend reaches `https://api.desineuron.in`
- ingress forwards to the current backend private IP
- backend public IP changes stop mattering
- Linux auto-heals route drift every 2 minutes and on boot

9
infrastructure/desineuron_ingress/desineuron-velocity-route-sync.service Normal file
View File

@@ -0,0 +1,9 @@
[Unit]
Description=Sync api.desineuron.in managed route to current Velocity backend private IP
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
EnvironmentFile=/etc/desineuron-velocity-route-sync.env
ExecStart=/opt/desineuron-velocity-route-sync/.venv/bin/python /usr/local/bin/sync_velocity_route.py

10
infrastructure/desineuron_ingress/desineuron-velocity-route-sync.timer Normal file
View File

@@ -0,0 +1,10 @@
[Unit]
Description=Run velocity route sync on boot and every 2 minutes
[Timer]
OnBootSec=1min
OnUnitActiveSec=2min
Unit=desineuron-velocity-route-sync.service
[Install]
WantedBy=timers.target

33
infrastructure/desineuron_ingress/install_linux_velocity_route_sync.sh Normal file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euo pipefail
APP_ROOT=/opt/desineuron-velocity-route-sync
VENV_PATH="$APP_ROOT/.venv"
ENV_FILE=/etc/desineuron-velocity-route-sync.env
SCRIPT_PATH=/usr/local/bin/sync_velocity_route.py
SERVICE_FILE=/etc/systemd/system/desineuron-velocity-route-sync.service
TIMER_FILE=/etc/systemd/system/desineuron-velocity-route-sync.timer
sudo mkdir -p "$APP_ROOT" /var/lib/desineuron-velocity-route-sync
python3 -m venv "$VENV_PATH"
"$VENV_PATH/bin/pip" install --upgrade pip boto3
sudo install -m 0755 /tmp/desineuron_ingress/sync_velocity_route.py "$SCRIPT_PATH"
sudo install -m 0644 /tmp/desineuron_ingress/desineuron-velocity-route-sync.service "$SERVICE_FILE"
sudo install -m 0644 /tmp/desineuron_ingress/desineuron-velocity-route-sync.timer "$TIMER_FILE"
sudo tee "$ENV_FILE" >/dev/null <<EOF
OPS_ENV_FILE=/opt/desineuron-ops-control-plane/.env
VELOCITY_ROUTE_HOSTNAME=api.desineuron.in
VELOCITY_ROUTE_PORT=8001
VELOCITY_INSTANCE_TAG_KEY=DesineuronRole
VELOCITY_INSTANCE_TAG_VALUE=velocity-backend
VELOCITY_ROUTE_STATE_FILE=/var/lib/desineuron-velocity-route-sync/current_target.txt
INGRESS_SSH_KEY_PATH=/opt/desineuron-ops-control-plane/state/desineuron-l4-node.pem
EOF
sudo chmod 600 "$ENV_FILE"
sudo systemctl daemon-reload
sudo systemctl enable --now desineuron-velocity-route-sync.timer
sudo systemctl start desineuron-velocity-route-sync.service
sudo systemctl --no-pager --full status desineuron-velocity-route-sync.service desineuron-velocity-route-sync.timer

142
infrastructure/desineuron_ingress/sync_velocity_route.py Normal file
View File

@@ -0,0 +1,142 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import boto3
def load_env_file(path: Path) -> dict[str, str]:
data: dict[str, str] = {}
if not path.exists():
return data
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
data[key.strip()] = value.strip()
return data
def env(name: str, default: str = "") -> str:
return os.environ.get(name, default)
def resolve_target_instance(ec2) -> dict | None:
explicit_instance_id = env("VELOCITY_INSTANCE_ID")
if explicit_instance_id:
reservations = ec2.describe_instances(InstanceIds=[explicit_instance_id])["Reservations"]
for reservation in reservations:
for instance in reservation["Instances"]:
if instance["State"]["Name"] == "running":
return instance
return None
tag_key = env("VELOCITY_INSTANCE_TAG_KEY", "DesineuronRole")
tag_value = env("VELOCITY_INSTANCE_TAG_VALUE", "velocity-backend")
filters = [
{"Name": "instance-state-name", "Values": ["running"]},
{"Name": f"tag:{tag_key}", "Values": [tag_value]},
]
reservations = ec2.describe_instances(Filters=filters)["Reservations"]
instances = [instance for reservation in reservations for instance in reservation["Instances"]]
if not instances:
return None
instances.sort(key=lambda row: row["LaunchTime"], reverse=True)
return instances[0]
def upsert_route(hostname: str, private_ip: str, port: int) -> subprocess.CompletedProcess[str]:
ingress_host = env("INGRESS_SSH_HOST")
ingress_user = env("INGRESS_SSH_USER", "ec2-user")
ingress_port = env("INGRESS_SSH_PORT", "22")
ingress_key = env("INGRESS_SSH_KEY_PATH")
helper = env("INGRESS_ROUTE_HELPER", "/usr/local/bin/manage_desineuron_routes.py")
payload = json.dumps(
{
"hostname": hostname,
"scheme": "http",
"target_host": private_ip,
"target_port": port,
}
)
command = (
f"sudo {helper} upsert '{payload}'"
" && sudo caddy validate --config /etc/caddy/Caddyfile"
" && sudo systemctl reload caddy"
)
return subprocess.run(
[
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-i",
ingress_key,
"-p",
ingress_port,
f"{ingress_user}@{ingress_host}",
command,
],
capture_output=True,
text=True,
check=False,
)
def main() -> int:
ops_env = load_env_file(Path(env("OPS_ENV_FILE", "/opt/desineuron-ops-control-plane/.env")))
for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_DEFAULT_REGION"]:
if key not in os.environ and key in ops_env:
os.environ[key] = ops_env[key]
os.environ.setdefault("AWS_DEFAULT_REGION", ops_env.get("OPS_DEFAULT_REGION", "us-east-1"))
os.environ.setdefault("INGRESS_SSH_HOST", ops_env.get("OPS_INGRESS_SSH_HOST", ""))
os.environ.setdefault("INGRESS_SSH_USER", ops_env.get("OPS_INGRESS_SSH_USER", "ec2-user"))
os.environ.setdefault("INGRESS_SSH_PORT", ops_env.get("OPS_INGRESS_SSH_PORT", "22"))
normalized_key_path = ops_env.get("OPS_SSH_KEY_PATH", "/opt/desineuron-ops-control-plane/state/desineuron-l4-node.pem")
if normalized_key_path.startswith("/app/state/"):
normalized_key_path = normalized_key_path.replace("/app/state/", "/opt/desineuron-ops-control-plane/state/")
os.environ.setdefault("INGRESS_SSH_KEY_PATH", normalized_key_path)
os.environ.setdefault("INGRESS_ROUTE_HELPER", ops_env.get("OPS_INGRESS_ROUTE_HELPER", "/usr/local/bin/manage_desineuron_routes.py"))
region = os.environ["AWS_DEFAULT_REGION"]
hostname = env("VELOCITY_ROUTE_HOSTNAME", "api.desineuron.in")
port = int(env("VELOCITY_ROUTE_PORT", "8001"))
state_file = Path(env("VELOCITY_ROUTE_STATE_FILE", "/var/lib/desineuron-velocity-route-sync/current_target.txt"))
ec2 = boto3.client("ec2", region_name=region)
instance = resolve_target_instance(ec2)
if not instance:
print("No running velocity-backend target instance found", file=sys.stderr)
return 1
private_ip = instance.get("PrivateIpAddress")
if not private_ip:
print("Target instance has no private IP", file=sys.stderr)
return 1
current = state_file.read_text(encoding="utf-8").strip() if state_file.exists() else ""
if current == private_ip:
print(json.dumps({"status": "noop", "hostname": hostname, "target_host": private_ip}))
return 0
result = upsert_route(hostname, private_ip, port)
if result.returncode != 0:
print(result.stdout)
print(result.stderr, file=sys.stderr)
return result.returncode
state_file.parent.mkdir(parents=True, exist_ok=True)
state_file.write_text(private_ip, encoding="utf-8")
print(json.dumps({"status": "updated", "hostname": hostname, "target_host": private_ip}))
return 0
if __name__ == "__main__":
raise SystemExit(main())

2
remote_bootstrap_20260401.sh
View File

@@ -34,7 +34,7 @@ VELOCITY_DB_NAME=velocity
VELOCITY_DB_USER=velocity_app VELOCITY_DB_USER=velocity_app
VELOCITY_DB_PASSWORD=${DB_PASSWORD} VELOCITY_DB_PASSWORD=${DB_PASSWORD}
VELOCITY_JWT_SECRET=${JWT_SECRET} VELOCITY_JWT_SECRET=${JWT_SECRET}
CORS_ORIGINS=http://localhost:5173,https://18.212.122.77 CORS_ORIGINS=http://localhost:5173,https://api.desineuron.in,https://54.152.236.10,https://18.212.122.77
VELOCITY_ASSET_DIR=/opt/dlami/nvme/assets VELOCITY_ASSET_DIR=/opt/dlami/nvme/assets
OLLAMA_BASE_URL=http://127.0.0.1:11434 OLLAMA_BASE_URL=http://127.0.0.1:11434
NEMOCLAW_BASE_URL=http://127.0.0.1:8080 NEMOCLAW_BASE_URL=http://127.0.0.1:8080