sagnik/Project_Velocity
2
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix/#22 (#23)

Browse Source 
Co-authored-by: Sagnik <sagnik7896@gmail.com>
Reviewed-on: #23
This commit was merged in pull request #23.
This commit is contained in:
sayan
2026-04-13 02:35:24 +05:30
parent bf950bc789
commit 857e0b88e6
13 changed files with 26773 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
app/src/components/modules/sentinel/PerceptionPlayer.tsx
View File

@@ -41,7 +41,8 @@ import { useVelocitySocket } from '@/hooks/useVelocitySocket';
import { encodeLandmarkPacket, hasSignificantActivity } from '@/utils/landmarkPacketEncoder';
import { API_URL } from '@/lib/api';
import { cn } from '@/lib/utils';
import type { QDScoreUpdate } from '@/types';
import { useStore } from '@/store/useStore';
import type { Lead, QDScoreUpdate } from '@/types';
// ── Types ─────────────────────────────────────────────────────────────────────
@@ -57,6 +58,13 @@ interface PerceptionPlayerProps {
const SPEEDS = [0.5, 0.75, 1, 1.5, 2] as const;
type Speed = (typeof SPEEDS)[number];
const PERCEPTION_CAMERA_CONSTRAINTS: MediaTrackConstraints = {
width: { ideal: 1280, min: 640 },
height: { ideal: 720, min: 480 },
frameRate: { ideal: 30, min: 24 },
facingMode: 'user',
};
// ── Helpers ───────────────────────────────────────────────────────────────────
function fmt(s: number): string {
@@ -194,6 +202,146 @@ function VolumeControl({
// ── Main Component ────────────────────────────────────────────────────────────
function clampScore(value: number): number {
return Math.max(1, Math.min(100, Math.round(value)));
}
function buildScoreMap(categories: Array<{ categoryName: string; score: number }>): Record<string, number> {
const map: Record<string, number> = {};
for (const category of categories) {
map[category.categoryName] = category.score;
}
return map;
}
function deriveInitialSessionScore(lead?: Lead | null, priorInteractionCount = 0): number {
const baseScore = lead?.quantumDynamicsScore ?? 50;
let modifier = 0;
const budget = (lead?.budget ?? '').toLowerCase();
const tags = (lead?.tags ?? []).map((tag) => tag.toLowerCase());
if (/(10m|15m|20m|crore|million)/.test(budget)) {
modifier += 15;
} else if (/(5m|8m)/.test(budget)) {
modifier += 8;
}
if (tags.includes('hni')) modifier += 12;
if (tags.includes('nri')) modifier += 5;
if (baseScore > 75) {
modifier += 8;
} else if (baseScore > 50) {
modifier += 4;
}
if (priorInteractionCount > 5) {
modifier += 8;
} else if (priorInteractionCount >= 2) {
modifier += 4;
}
return clampScore(baseScore + modifier);
}
function calculateLocalQdScore(previousScore: number, scoreMap: Record<string, number>): number {
const smileLeft = scoreMap.mouthSmileLeft ?? 0;
const smileRight = scoreMap.mouthSmileRight ?? 0;
const smileMax = Math.max(smileLeft, smileRight);
const browInnerUp = scoreMap.browInnerUp ?? 0;
const eyeWideLeft = scoreMap.eyeWideLeft ?? 0;
const eyeWideRight = scoreMap.eyeWideRight ?? 0;
const eyeWideMax = Math.max(eyeWideLeft, eyeWideRight);
const jawOpen = scoreMap.jawOpen ?? 0;
const cheekPuff = scoreMap.cheekPuff ?? 0;
const browDownLeft = scoreMap.browDownLeft ?? 0;
const browDownRight = scoreMap.browDownRight ?? 0;
const eyeBlinkLeft = scoreMap.eyeBlinkLeft ?? 0;
const eyeBlinkRight = scoreMap.eyeBlinkRight ?? 0;
const mouthFrown = Math.max(scoreMap.mouthFrownLeft ?? 0, scoreMap.mouthFrownRight ?? 0);
let positiveDelta = 0;
if (smileLeft > 0.5 && smileRight > 0.5) positiveDelta += 15;
else if (smileLeft > 0.5 || smileRight > 0.5) positiveDelta += 10;
if (browInnerUp > 0.4) positiveDelta += 8;
if (eyeWideMax > 0.5) positiveDelta += 7;
if (jawOpen > 0.3 && eyeWideMax > 0.5) positiveDelta += 5;
if (cheekPuff > 0.3) positiveDelta += 3;
let negativeDelta = 0;
if (browDownLeft > 0.45 && browDownRight > 0.45 && smileMax < 0.2) negativeDelta -= 10;
if (eyeBlinkLeft > 0.7 && eyeBlinkRight > 0.7 && eyeWideMax < 0.2) negativeDelta -= 15;
if (mouthFrown > 0.4) negativeDelta -= 8;
const weightedShapes = [
smileLeft,
smileRight,
browInnerUp,
eyeWideLeft,
eyeWideRight,
jawOpen,
cheekPuff,
browDownLeft,
browDownRight,
eyeBlinkLeft,
eyeBlinkRight,
mouthFrown,
];
if (weightedShapes.every((value) => value < 0.15)) negativeDelta -= 3;
let delta = positiveDelta + negativeDelta;
if (positiveDelta > 0 && negativeDelta < 0) {
delta = Math.abs(positiveDelta) >= Math.abs(negativeDelta) ? positiveDelta : negativeDelta;
}
const boundedDelta = Math.max(-20, Math.min(20, delta));
return clampScore(previousScore + boundedDelta);
}
function getFaceBounds(
landmarks: Array<{ x: number; y: number; z: number }>,
containerWidth: number,
containerHeight: number,
sourceWidth: number,
sourceHeight: number,
mirrored = true,
) {
const safeSourceWidth = sourceWidth > 0 ? sourceWidth : containerWidth;
const safeSourceHeight = sourceHeight > 0 ? sourceHeight : containerHeight;
const scale = Math.max(containerWidth / safeSourceWidth, containerHeight / safeSourceHeight);
const renderedWidth = safeSourceWidth * scale;
const renderedHeight = safeSourceHeight * scale;
const offsetX = (containerWidth - renderedWidth) / 2;
const offsetY = (containerHeight - renderedHeight) / 2;
let minX = Infinity;
let minY = Infinity;
let maxX = -Infinity;
let maxY = -Infinity;
for (const point of landmarks) {
const rawX = offsetX + (point.x * renderedWidth);
const rawY = offsetY + (point.y * renderedHeight);
const x = mirrored ? containerWidth - rawX : rawX;
const y = rawY;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
const paddingX = Math.max(8, (maxX - minX) * 0.14);
const paddingY = Math.max(8, (maxY - minY) * 0.18);
return {
x: Math.max(0, minX - paddingX),
y: Math.max(0, minY - paddingY),
width: Math.min(containerWidth, (maxX - minX) + (paddingX * 2)),
height: Math.min(containerHeight, (maxY - minY) + (paddingY * 2)),
area: Math.max(0, (maxX - minX) * (maxY - minY)),
};
}
export function PerceptionPlayer({
videoUrl, videoAssetId, leadId,
sessionMode = 'assigned', videoTitle, onSessionComplete,
@@ -202,6 +350,9 @@ export function PerceptionPlayer({
const [sessionId] = useState(
() => globalThis.crypto?.randomUUID?.() ?? `00000000-0000-4000-8000-${reactId.slice(-12).padStart(12, '0')}`,
);
const lead = useStore((state) => state.leads.find((candidate) => candidate.id === leadId) ?? null);
const priorInteractionCount = useStore((state) => (leadId ? state.messages[leadId]?.length ?? 0 : 0));
const initialSessionScore = deriveInitialSessionScore(lead, priorInteractionCount);
// Refs
const videoRef = useRef<HTMLVideoElement>(null);
@@ -213,13 +364,15 @@ export function PerceptionPlayer({
const streamRef = useRef<MediaStream | null>(null);
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const scrubbing = useRef(false);
const qdScoreRef = useRef(initialSessionScore);
const lastBackendQdTsRef = useRef<number>(0);
// State — session
const [consentGiven, setConsentGiven] = useState(false);
const [cameraActive, setCameraActive] = useState(false);
const [cameraError, setCameraError] = useState<string | null>(null);
const [sessionActive, setSessionActive] = useState(false);
const [qdScore, setQdScore] = useState(50);
const [qdScore, setQdScore] = useState(initialSessionScore);
const [showCamera, setShowCamera] = useState(true);
// State — playback
@@ -246,10 +399,25 @@ export function PerceptionPlayer({
if (msg.type !== 'QD_UPDATED') return;
const d = msg.data as Partial<QDScoreUpdate>;
const match = (leadId && d.lead_id === leadId) || d.session_id === sessionId;
if (match && typeof d.qd_score === 'number') setQdScore(d.qd_score);
if (match && typeof d.qd_score === 'number') {
lastBackendQdTsRef.current = Date.now();
qdScoreRef.current = d.qd_score;
setQdScore(d.qd_score);
}
}, [leadId, sessionId]),
});
useEffect(() => {
qdScoreRef.current = initialSessionScore;
setQdScore(initialSessionScore);
}, [initialSessionScore, leadId, sessionId]);
useEffect(() => {
if (!cameraActive || !streamRef.current || !webcamRef.current) return;
webcamRef.current.srcObject = streamRef.current;
void webcamRef.current.play().catch(() => null);
}, [cameraActive]);
// ── Auto-hide (YouTube behaviour) ─────────────────────────────────────────
// Show controls, reset the hide timer. After 2.8 s of no movement while
// playing, hide controls and the cursor.
@@ -289,14 +457,10 @@ export function PerceptionPlayer({
const startCamera = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 320, height: 240, facingMode: 'user' },
video: PERCEPTION_CAMERA_CONSTRAINTS,
audio: false,
});
streamRef.current = stream;
if (webcamRef.current) {
webcamRef.current.srcObject = stream;
await webcamRef.current.play();
}
setCameraActive(true);
setCameraError(null);
} catch (err) {
@@ -325,39 +489,56 @@ export function PerceptionPlayer({
const videoTsMs = Math.round(video.currentTime * 1000);
const result = detectFrame(webcam, now);
if (!result?.faceBlendshapes?.length) return;
if (!result?.faceLandmarks?.length) return;
const categories = result.faceBlendshapes[0]?.categories ?? [];
const landmarkSets = result.faceLandmarks ?? [];
const overlayWidth = canvasRef.current?.width ?? 112;
const overlayHeight = canvasRef.current?.height ?? 80;
const sourceWidth = webcam.videoWidth || overlayWidth;
const sourceHeight = webcam.videoHeight || overlayHeight;
const boundsByFace = landmarkSets.map((landmarks) =>
getFaceBounds(landmarks, overlayWidth, overlayHeight, sourceWidth, sourceHeight, true),
);
const primaryFaceIndex = boundsByFace.reduce((bestIndex, bounds, index, allBounds) => (
bounds.area > allBounds[bestIndex].area ? index : bestIndex
), 0);
const categories = result.faceBlendshapes?.[primaryFaceIndex]?.categories ?? [];
const localScoreMap = buildScoreMap(categories);
const packet = encodeLandmarkPacket(categories, videoTsMs, leadId ?? '', sessionId);
if (!hasSignificantActivity(packet)) return;
const significantActivity = hasSignificantActivity(packet);
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
if (significantActivity) {
sendPacket({
event: 'BIOMETRIC_PACKET',
lead_id: leadId, session_id: sessionId,
session_mode: sessionMode, video_asset_id: videoAssetId,
video_ts_ms: packet.video_ts_ms, blend_shapes: packet.blend_shapes,
});
}
if (Date.now() - lastBackendQdTsRef.current > 2500) {
const rawScore = calculateLocalQdScore(qdScoreRef.current, localScoreMap);
const smoothedScore = clampScore((qdScoreRef.current * 0.7) + (rawScore * 0.3));
qdScoreRef.current = smoothedScore;
setQdScore(smoothedScore);
}
const canvas = canvasRef.current;
if (canvas && result.faceLandmarks?.length) {
if (canvas && landmarkSets.length) {
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
const lm = result.faceLandmarks[0];
if (lm?.length) {
const pts = [10, 234, 454, 152, 58, 288];
ctx.strokeStyle = 'rgba(59,130,246,0.7)';
ctx.lineWidth = 1.5;
boundsByFace.forEach((bounds, index) => {
const isPrimary = index === primaryFaceIndex;
ctx.strokeStyle = isPrimary ? 'rgba(59,130,246,0.92)' : 'rgba(255,255,255,0.5)';
ctx.lineWidth = isPrimary ? 2 : 1.2;
ctx.fillStyle = isPrimary ? 'rgba(59,130,246,0.12)' : 'rgba(255,255,255,0.06)';
ctx.beginPath();
pts.forEach((idx, i) => {
const p = lm[idx];
if (!p) return;
i === 0 ? ctx.moveTo(p.x * canvas.width, p.y * canvas.height)
: ctx.lineTo(p.x * canvas.width, p.y * canvas.height);
});
ctx.closePath();
ctx.roundRect(bounds.x, bounds.y, bounds.width, bounds.height, 8);
ctx.fill();
ctx.stroke();
}
});
}
}
}, [detectFrame, isReady, leadId, sendPacket, sessionId, sessionMode, videoAssetId]);