feat: Complete code integration of modules (#18)

The complete code integration is done.

Co-authored-by: Sagnik <sagnik7896@gmail.com>
Reviewed-on: #18

backend/oracle/action_service.py

@@ -0,0 +1,346 @@
+from __future__ import annotations
+
+import json
+import os
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import HTTPException
+
+try:
+    import asyncpg  # type: ignore
+except Exception:  # pragma: no cover
+    asyncpg = None  # type: ignore
+
+
+_DB_URL = os.getenv("DATABASE_URL", "")
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _db_ready() -> bool:
+    return bool(_DB_URL and not _DB_URL.startswith("PLACEHOLDER") and asyncpg is not None)
+
+
+class OracleActionService:
+    async def ensure_schema(self) -> None:
+        if not _db_ready():
+            return
+        assert asyncpg is not None
+        conn = await asyncpg.connect(_DB_URL)
+        try:
+            await conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS oracle_actions (
+                    action_id UUID PRIMARY KEY,
+                    execution_id UUID,
+                    tenant_id TEXT NOT NULL,
+                    page_id UUID,
+                    branch_id TEXT,
+                    actor_id TEXT NOT NULL,
+                    target_entity_type TEXT NOT NULL,
+                    target_entity_id TEXT,
+                    action_type TEXT NOT NULL,
+                    status TEXT NOT NULL DEFAULT 'planned',
+                    prompt TEXT,
+                    workflow_dispatch JSONB NOT NULL DEFAULT '{}'::jsonb,
+                    component_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
+                    writeback_payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+                    result_payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+                    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+                )
+                """
+            )
+            await conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_oracle_actions_execution ON oracle_actions(execution_id, created_at DESC)"
+            )
+            await conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_oracle_actions_target ON oracle_actions(target_entity_type, target_entity_id, created_at DESC)"
+            )
+        finally:
+            await conn.close()
+
+    async def create_from_execution(
+        self,
+        *,
+        execution: dict[str, Any],
+        target_entity_type: str = "canvas_page",
+        target_entity_id: str | None = None,
+        action_type: str = "oracle_canvas_generation",
+        writeback_payload: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        action = {
+            "actionId": str(uuid.uuid4()),
+            "executionId": execution.get("executionId"),
+            "tenantId": execution.get("tenantId"),
+            "pageId": execution.get("pageId"),
+            "branchId": execution.get("branchId"),
+            "actorId": execution.get("actorId"),
+            "targetEntityType": target_entity_type,
+            "targetEntityId": target_entity_id or execution.get("pageId"),
+            "actionType": action_type,
+            "status": "planned",
+            "prompt": execution.get("prompt"),
+            "workflowDispatch": execution.get("workflowDispatch") or {},
+            "componentIds": execution.get("componentsCreated") or [],
+            "writebackPayload": writeback_payload or {},
+            "resultPayload": {},
+            "createdAt": _now(),
+            "updatedAt": _now(),
+        }
+        await self._persist_action(action)
+        return action
+
+    async def get_action(self, action_id: str) -> dict[str, Any] | None:
+        if not _db_ready():
+            return None
+        assert asyncpg is not None
+        conn = await asyncpg.connect(_DB_URL)
+        try:
+            row = await conn.fetchrow(
+                """
+                SELECT action_id, execution_id, tenant_id, page_id, branch_id, actor_id,
+                       target_entity_type, target_entity_id, action_type, status, prompt,
+                       workflow_dispatch, component_ids, writeback_payload, result_payload,
+                       created_at, updated_at
+                FROM oracle_actions
+                WHERE action_id = $1::uuid
+                """,
+                action_id,
+            )
+        finally:
+            await conn.close()
+        return self._serialize(row) if row else None
+
+    async def list_actions(self, *, status: str | None = None, limit: int = 50) -> list[dict[str, Any]]:
+        if not _db_ready():
+            return []
+        assert asyncpg is not None
+        conn = await asyncpg.connect(_DB_URL)
+        try:
+            if status:
+                rows = await conn.fetch(
+                    """
+                    SELECT action_id, execution_id, tenant_id, page_id, branch_id, actor_id,
+                           target_entity_type, target_entity_id, action_type, status, prompt,
+                           workflow_dispatch, component_ids, writeback_payload, result_payload,
+                           created_at, updated_at
+                    FROM oracle_actions
+                    WHERE status = $1
+                    ORDER BY created_at DESC
+                    LIMIT $2
+                    """,
+                    status,
+                    limit,
+                )
+            else:
+                rows = await conn.fetch(
+                    """
+                    SELECT action_id, execution_id, tenant_id, page_id, branch_id, actor_id,
+                           target_entity_type, target_entity_id, action_type, status, prompt,
+                           workflow_dispatch, component_ids, writeback_payload, result_payload,
+                           created_at, updated_at
+                    FROM oracle_actions
+                    ORDER BY created_at DESC
+                    LIMIT $1
+                    """,
+                    limit,
+                )
+        finally:
+            await conn.close()
+        return [self._serialize(row) for row in rows]
+
+    async def apply_writeback(self, payload: dict[str, Any]) -> dict[str, Any]:
+        if not _db_ready():
+            raise HTTPException(status_code=503, detail="Oracle writeback store unavailable.")
+        if payload["target_entity_type"] != "lead":
+            raise HTTPException(status_code=422, detail="Only lead writebacks are supported in this pass.")
+
+        assert asyncpg is not None
+        await self.ensure_schema()
+        conn = await asyncpg.connect(_DB_URL)
+        try:
+            target_lead_id = payload["target_entity_id"]
+            action_id = payload["action_id"]
+            writeback = payload["writeback_payload"]
+
+            existing = await conn.fetchrow(
+                "SELECT id, notes, metadata, kanban_status, qualification, score FROM leads WHERE id = $1",
+                target_lead_id,
+            )
+            if existing is None:
+                raise HTTPException(status_code=404, detail=f"Lead '{target_lead_id}' not found for Oracle writeback.")
+
+            metadata = dict(existing["metadata"] or {})
+            metadata_patch = writeback.get("metadata_patch") or {}
+            if isinstance(metadata_patch, dict):
+                metadata.update(metadata_patch)
+
+            score = int(existing["score"] or 0) + int(writeback.get("score_delta") or 0)
+            updated_notes = (existing["notes"] or "").strip()
+            notes_append = writeback.get("notes_append")
+            if notes_append:
+                separator = "\n\n" if updated_notes else ""
+                updated_notes = f"{updated_notes}{separator}{notes_append}"
+
+            updated = await conn.fetchrow(
+                """
+                UPDATE leads
+                SET notes = $2,
+                    metadata = $3::jsonb,
+                    kanban_status = COALESCE($4, kanban_status),
+                    qualification = COALESCE($5, qualification),
+                    score = $6,
+                    updated_at = NOW()
+                WHERE id = $1
+                RETURNING id, notes, metadata, kanban_status, qualification, score, updated_at
+                """,
+                target_lead_id,
+                updated_notes,
+                json.dumps(metadata),
+                writeback.get("kanban_status"),
+                writeback.get("qualification"),
+                max(score, 0),
+            )
+
+            oracle_message = writeback.get("oracle_message")
+            if oracle_message:
+                await conn.execute(
+                    """
+                    INSERT INTO chat_logs (id, lead_id, sender, channel, content, metadata, created_at)
+                    VALUES ($1, $2, 'oracle', 'oracle', $3, $4::jsonb, NOW())
+                    """,
+                    str(uuid.uuid4()),
+                    target_lead_id,
+                    oracle_message,
+                    json.dumps({"oracle_action_id": action_id, "writeback": True}),
+                )
+
+            result_payload = {
+                "lead_id": updated["id"],
+                "kanban_status": updated["kanban_status"],
+                "qualification": updated["qualification"],
+                "score": updated["score"],
+                "updated_at": updated["updated_at"].isoformat() if updated["updated_at"] else None,
+            }
+
+            await conn.execute(
+                """
+                INSERT INTO oracle_actions (
+                    action_id, execution_id, tenant_id, page_id, branch_id, actor_id,
+                    target_entity_type, target_entity_id, action_type, status, prompt,
+                    workflow_dispatch, component_ids, writeback_payload, result_payload,
+                    created_at, updated_at
+                )
+                VALUES (
+                    $1::uuid, NULL, $2, NULL, NULL, $3,
+                    $4, $5, $6, 'applied', NULL,
+                    '{}'::jsonb, '[]'::jsonb, $7::jsonb, $8::jsonb,
+                    NOW(), NOW()
+                )
+                ON CONFLICT (action_id)
+                DO UPDATE SET
+                    status = 'applied',
+                    writeback_payload = EXCLUDED.writeback_payload,
+                    result_payload = EXCLUDED.result_payload,
+                    updated_at = NOW()
+                """,
+                action_id,
+                payload.get("tenant_id", "tenant_velocity"),
+                payload.get("actor_id", "oracle_operator"),
+                payload["target_entity_type"],
+                target_lead_id,
+                payload.get("action_type", "lead_writeback"),
+                json.dumps(writeback),
+                json.dumps(result_payload),
+            )
+        finally:
+            await conn.close()
+
+        return {
+            "actionId": action_id,
+            "status": "applied",
+            "targetEntityType": payload["target_entity_type"],
+            "targetEntityId": payload["target_entity_id"],
+            "resultPayload": result_payload,
+        }
+
+    async def _persist_action(self, action: dict[str, Any]) -> None:
+        if not _db_ready():
+            return
+        await self.ensure_schema()
+        assert asyncpg is not None
+        conn = await asyncpg.connect(_DB_URL)
+        try:
+            await conn.execute(
+                """
+                INSERT INTO oracle_actions (
+                    action_id, execution_id, tenant_id, page_id, branch_id, actor_id,
+                    target_entity_type, target_entity_id, action_type, status, prompt,
+                    workflow_dispatch, component_ids, writeback_payload, result_payload,
+                    created_at, updated_at
+                )
+                VALUES (
+                    $1::uuid, $2::uuid, $3, $4::uuid, $5, $6,
+                    $7, $8, $9, $10, $11,
+                    $12::jsonb, $13::jsonb, $14::jsonb, $15::jsonb,
+                    $16::timestamptz, $17::timestamptz
+                )
+                ON CONFLICT (action_id)
+                DO UPDATE SET
+                    status = EXCLUDED.status,
+                    workflow_dispatch = EXCLUDED.workflow_dispatch,
+                    component_ids = EXCLUDED.component_ids,
+                    writeback_payload = EXCLUDED.writeback_payload,
+                    result_payload = EXCLUDED.result_payload,
+                    updated_at = EXCLUDED.updated_at
+                """,
+                action["actionId"],
+                action.get("executionId"),
+                action["tenantId"],
+                action.get("pageId"),
+                action.get("branchId"),
+                action["actorId"],
+                action["targetEntityType"],
+                action.get("targetEntityId"),
+                action["actionType"],
+                action["status"],
+                action.get("prompt"),
+                json.dumps(action.get("workflowDispatch") or {}),
+                json.dumps(action.get("componentIds") or []),
+                json.dumps(action.get("writebackPayload") or {}),
+                json.dumps(action.get("resultPayload") or {}),
+                action["createdAt"],
+                action["updatedAt"],
+            )
+        finally:
+            await conn.close()
+
+    @staticmethod
+    def _serialize(row: Any) -> dict[str, Any]:
+        return {
+            "actionId": str(row["action_id"]),
+            "executionId": str(row["execution_id"]) if row["execution_id"] else None,
+            "tenantId": row["tenant_id"],
+            "pageId": str(row["page_id"]) if row["page_id"] else None,
+            "branchId": row["branch_id"],
+            "actorId": row["actor_id"],
+            "targetEntityType": row["target_entity_type"],
+            "targetEntityId": row["target_entity_id"],
+            "actionType": row["action_type"],
+            "status": row["status"],
+            "prompt": row["prompt"],
+            "workflowDispatch": row["workflow_dispatch"] or {},
+            "componentIds": row["component_ids"] or [],
+            "writebackPayload": row["writeback_payload"] or {},
+            "resultPayload": row["result_payload"] or {},
+            "createdAt": row["created_at"].isoformat() if row["created_at"] else None,
+            "updatedAt": row["updated_at"].isoformat() if row["updated_at"] else None,
+        }
+
+
+oracle_action_service = OracleActionService()

backend/oracle/persona_service.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+_PROMPT_DIR = Path(__file__).resolve().parent.parent / "nemoclaw_prompts"
+_PLACEHOLDER_PATTERN = re.compile(r"\{(\w+)\}")
+_TEMPLATE_HINTS = {
+    "pipeline": ["tpl_pipeline_board_v2", "tpl_followup_queue_v1"],
+    "kanban": ["tpl_pipeline_board_v2"],
+    "map": ["tpl_geo_investor_heat_v2"],
+    "geo": ["tpl_geo_investor_heat_v2"],
+    "trend": ["tpl_absorption_trend_v1", "tpl_campaign_lead_line_v1"],
+    "quota": ["tpl_quota_gauge_v1", "tpl_kpi_pipeline_health_v1"],
+    "broker": ["tpl_broker_performance_v1"],
+    "source": ["tpl_qd_source_compare_v1", "tpl_bar_source_quality_v3"],
+    "follow": ["tpl_followup_queue_v1", "tpl_followup_gap_v1"],
+    "campaign": ["tpl_campaign_lead_line_v1"],
+}
+
+
+class PersonaService:
+    def __init__(self) -> None:
+        self.prompt_files = {
+            "qd_calculator": _PROMPT_DIR / "qd_calculator.md",
+            "lead_tagger": _PROMPT_DIR / "lead_tagger.md",
+            "cctv_profiler": _PROMPT_DIR / "cctv_profiler.md",
+        }
+
+    async def health(self) -> dict[str, Any]:
+        loaded = {}
+        for key, path in self.prompt_files.items():
+            loaded[key] = path.exists() and path.read_text(encoding="utf-8").strip() != ""
+        return {
+            "status": "healthy" if all(loaded.values()) else "degraded",
+            "prompts": loaded,
+        }
+
+    async def render_prompt(
+        self,
+        *,
+        prompt_name: str,
+        variables: dict[str, Any],
+    ) -> dict[str, Any]:
+        path = self.prompt_files.get(prompt_name)
+        if path is None or not path.exists():
+            raise FileNotFoundError(f"Unknown prompt '{prompt_name}'.")
+        template = path.read_text(encoding="utf-8")
+        rendered = template
+        for key, value in variables.items():
+            rendered = rendered.replace(f"{{{key}}}", json.dumps(value) if isinstance(value, (dict, list)) else str(value))
+        unresolved = sorted(set(_PLACEHOLDER_PATTERN.findall(rendered)))
+        return {
+            "promptName": prompt_name,
+            "templatePath": str(path),
+            "renderedPrompt": rendered,
+            "unresolvedVariables": unresolved,
+        }
+
+    async def plan_for_prompt(
+        self,
+        *,
+        prompt: str,
+        tenant_id: str,
+        actor_role: str,
+    ) -> dict[str, Any]:
+        lower_prompt = prompt.lower()
+        recommended: list[str] = []
+        for token, template_ids in _TEMPLATE_HINTS.items():
+            if token in lower_prompt:
+                recommended.extend(template_ids)
+        if not recommended:
+            recommended = ["tpl_kpi_pipeline_health_v1", "tpl_qd_source_compare_v1"]
+        recommended = list(dict.fromkeys(recommended))
+        return {
+            "tenantId": tenant_id,
+            "actorRole": actor_role,
+            "recommendedTemplates": recommended,
+            "canvasBlocks": [
+                {
+                    "type": "textCanvas",
+                    "widthMode": "full",
+                    "minHeightPx": 180,
+                    "content": (
+                        "Oracle planned a mixed response: query the CRM, reuse matching component templates, "
+                        "and synthesize missing visualization blocks if a direct template is unavailable."
+                    ),
+                }
+            ],
+            "workflowIntent": "comfy_oracle_canvas",
+        }
+
+
+persona_service = PersonaService()

backend/oracle/prompt_orchestrator.py

@@ -16,6 +16,8 @@ from typing import Any
 from .policy_service import PolicyContext, PolicyService
 from .canvas_service import canvas_service
 from .data_access_gateway import data_access_gateway
+from .persona_service import persona_service
+from backend.services.nemoclaw_runtime import nemoclaw_runtime
 
 try:
     import asyncpg  # type: ignore
@@ -177,6 +179,19 @@ class PromptOrchestrator:
 
         execution["retrievalPlan"] = retrieval_plan
 
+        persona_plan = await persona_service.plan_for_prompt(
+            prompt=prompt,
+            tenant_id=tenant_id,
+            actor_role=actor_role,
+        )
+        execution["personaPlan"] = persona_plan
+        execution["workflowDispatch"] = nemoclaw_runtime.build_workflow_dispatch(
+            prompt=prompt,
+            tenant_id=tenant_id,
+            actor_role=actor_role,
+            component_templates=persona_plan["recommendedTemplates"],
+        )
+
         # ── Step 2: Policy validation ─────────────────────────────────────────
         policy_errors = []
         for component_plan in retrieval_plan.get("components", []):
@@ -209,6 +224,7 @@ class PromptOrchestrator:
             branch_id=branch_id,
             placement_mode=placement_mode,
             ctx=ctx,
+            persona_plan=persona_plan,
         )
         execution["visualizationPlan"] = viz_plan
 
@@ -255,9 +271,18 @@ class PromptOrchestrator:
         branch_id: str,
         placement_mode: str,
         ctx: PolicyContext,
+        persona_plan: dict[str, Any],
     ) -> dict[str, Any]:
         """Converts a retrieval plan into a list of CanvasComponent descriptors."""
-        components = []
+        components = [
+            self._persona_text_canvas(
+                execution_id=execution_id,
+                actor_id=actor_id,
+                branch_id=branch_id,
+                prompt=prompt,
+                persona_plan=persona_plan,
+            )
+        ]
         base_order = 900  # Append after existing components
 
         component_plans = retrieval_plan.get("components", [])
@@ -343,6 +368,85 @@ class PromptOrchestrator:
 
         return {"components": components}
 
+    @staticmethod
+    def _persona_text_canvas(
+        *,
+        execution_id: str,
+        actor_id: str,
+        branch_id: str,
+        prompt: str,
+        persona_plan: dict[str, Any],
+    ) -> dict[str, Any]:
+        recommended = ", ".join(persona_plan.get("recommendedTemplates", [])) or "no direct template matches"
+        content = (
+            f"Oracle received: {prompt}\n\n"
+            f"Reusable templates: {recommended}\n\n"
+            "Execution policy: query live CRM data first, reuse matching templates, "
+            "synthesize missing UI blocks, then dispatch the required ComfyUI-backed workflow."
+        )
+        return {
+            "componentId": str(uuid.uuid4()),
+            "type": "textCanvas",
+            "title": "Oracle Planning Notes",
+            "description": "Persona-driven guidance generated before data-bound components.",
+            "dataSourceDescriptor": {
+                "descriptorId": str(uuid.uuid4()),
+                "sourceType": "inline",
+                "connectorId": "oracle-persona",
+                "dataset": "oracle_persona_plan",
+                "authContextRef": f"authctx_{actor_id}_scope",
+                "queryTemplate": "",
+                "queryParameters": {},
+                "rowLimit": 1,
+                "privacyTier": "standard",
+            },
+            "visualizationParameters": {
+                "content": content,
+                "widthMode": "full",
+                "adjustableHeight": True,
+            },
+            "dataBindings": {"dimensions": [], "measures": [], "series": [], "filters": []},
+            "version": 1,
+            "lifecycleState": "active",
+            "provenance": {
+                "originType": "prompt_generated",
+                "promptExecutionId": execution_id,
+                "sourceBranchId": branch_id,
+                "createdBy": actor_id,
+                "createdAt": _now(),
+            },
+            "renderingHints": {"estimatedHeightPx": 180, "skeletonVariant": "text", "virtualizationPriority": 4},
+            "layout": {
+                "orderIndex": 910,
+                "sectionId": "sec_prompt_generated",
+                "widthMode": "full",
+                "minHeightPx": 180,
+                "stickyHeader": False,
+            },
+            "accessControls": {
+                "visibilityScope": "private",
+                "allowedRoles": ["senior_broker", "sales_director", "marketing_operator", "data_steward", "compliance_reviewer", "platform_admin"],
+                "redactionPolicy": "none",
+            },
+            "styleSignature": {
+                "theme": "velocity_glass",
+                "paletteToken": "ocean_signal",
+                "motionProfile": "calm_reveal",
+                "density": "comfortable",
+                "radiusScale": "lg",
+                "typographyScale": "balanced",
+            },
+            "validationState": {
+                "schema": "pass",
+                "policy": "pass",
+                "a11y": "pass",
+                "performance": "pass",
+                "status": "validated",
+            },
+            "auditLog": [f"aud_{execution_id}_persona"],
+            "dataRows": [],
+        }
+
     @staticmethod
     def _map_type(plan_type: str) -> str:
         mapping = {

backend/oracle/router_v1.py

@@ -31,6 +31,8 @@ from pydantic import BaseModel, Field
 
 from .canvas_service import canvas_service
 from .collaboration_service import collaboration_service
+from .action_service import oracle_action_service
+from .persona_service import persona_service
 from .prompt_orchestrator import prompt_orchestrator
 from .policy_service import PolicyService, PolicyContext
 
@@ -96,6 +98,8 @@ class PromptSubmitRequest(BaseModel):
     prompt: str = Field(..., min_length=1, max_length=4096)
     conversationContext: list[dict[str, str]] = Field(default_factory=list)
     placementMode: str = Field("append_after_last_visible_component")
+    targetLeadId: str | None = None
+    plannedWriteback: dict[str, Any] = Field(default_factory=dict)
 
 
 class ForkCreateRequest(BaseModel):
@@ -131,6 +135,11 @@ class TemplateSynthesizeRequest(BaseModel):
     styleSignatureRef: str | None = None
 
 
+class PersonaRenderRequest(BaseModel):
+    promptName: str = Field(..., pattern="^(qd_calculator|lead_tagger|cctv_profiler)$")
+    variables: dict[str, Any] = Field(default_factory=dict)
+
+
 # ── Endpoints ─────────────────────────────────────────────────────────────────
 
 @router.get("/me", summary="Get current user profile")
@@ -167,8 +176,16 @@ async def submit_prompt(page_id: str, payload: PromptSubmitRequest) -> dict:
             detail={"errors": execution.get("warnings", [])},
         )
     page = await canvas_service.get_page(page_id, ctx.tenant_id)
+    action = await oracle_action_service.create_from_execution(
+        execution=execution,
+        target_entity_type="lead" if payload.targetLeadId else "canvas_page",
+        target_entity_id=payload.targetLeadId or page_id,
+        action_type="oracle_prompt_writeback_plan" if payload.targetLeadId else "oracle_canvas_generation",
+        writeback_payload=payload.plannedWriteback,
+    )
     return _ok({
         "executionId": execution["executionId"],
+        "actionId": action["actionId"],
         "status": execution["status"],
         "pageId": page_id,
         "branchId": payload.branchId,
@@ -250,6 +267,23 @@ async def synthesize_template(payload: TemplateSynthesizeRequest) -> dict:
     return _ok(template)
 
 
+@router.get("/persona/health", summary="Health check for Oracle persona prompt loading")
+async def persona_health() -> dict:
+    return _ok(await persona_service.health())
+
+
+@router.post("/persona/render", summary="Render a subordinate Oracle persona prompt")
+async def persona_render(payload: PersonaRenderRequest) -> dict:
+    try:
+        rendered = await persona_service.render_prompt(
+            prompt_name=payload.promptName,
+            variables=payload.variables,
+        )
+    except FileNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    return _ok(rendered)
+
+
 @router.get("/merge-requests", summary="List merge requests for a target page")
 async def list_merge_requests(targetPageId: str | None = None, status: str | None = None) -> dict:
     if not targetPageId: