feat: Ipad app production readiness, Colony orchestration, Social posting (#44)

#38 Ipad app production readiness, Colony orchestration, Social posting

Co-authored-by: Sayan Datta <sayan@Sayans-MacBook-Air.local>
Reviewed-on: #44

backend/api/routes_comms.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 
 from backend.auth.dependencies import UserPrincipal, get_current_user
 from backend.services.comms_evolution_provider import EvolutionProvider
-from backend.services.comms_ingest import ingest_inbound_message
+from backend.services.comms_ingest import TranscriptionError, ingest_inbound_message, transcribe_recording as run_transcription
 from backend.services.comms_provider import MockProvider
 from backend.services.comms_waha_provider import WahaProvider
 
@@ -46,6 +46,8 @@ class NoteBody(BaseModel):
 class TaskBody(BaseModel):
     title: str
     dueAt: str | None = None
+    notes: str | None = None
+    priority: str = "normal"
 
 
 class SettingsPatch(BaseModel):
@@ -158,6 +160,37 @@ def _record_value(row: Any, key: str, default: Any = None) -> Any:
         return default
 
 
+def _optional_datetime(value: str | None) -> datetime | None:
+    if not value or not value.strip():
+        return None
+    normalized = value.strip().replace("Z", "+00:00")
+    try:
+        return datetime.fromisoformat(normalized)
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail="dueAt must be an ISO-8601 timestamp.") from exc
+
+
+async def _thread_context(conn, thread_id: str, tenant_id: str):
+    thread = await conn.fetchrow("SELECT * FROM comms_threads WHERE thread_id = $1::uuid", thread_id)
+    if not thread:
+        raise HTTPException(status_code=404, detail="Thread not found")
+    lead_id = None
+    if thread["person_id"]:
+        lead_id = await conn.fetchval(
+            """
+            SELECT lead_id
+            FROM crm_leads
+            WHERE person_id = $1::uuid
+              AND tenant_id = $2
+            ORDER BY updated_at DESC NULLS LAST, created_at DESC NULLS LAST
+            LIMIT 1
+            """,
+            thread["person_id"],
+            tenant_id,
+        )
+    return thread, lead_id
+
+
 async def _ensure_schema(pool) -> None:
     global _SCHEMA_READY
     if _SCHEMA_READY:
@@ -182,6 +215,19 @@ async def _ensure_schema(pool) -> None:
                 created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
                 updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
             );
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'mock';
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS external_thread_id TEXT;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS person_id UUID NULL REFERENCES crm_people(person_id) ON DELETE SET NULL;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS phone_e164 TEXT NOT NULL DEFAULT '';
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS display_name TEXT;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS channel TEXT NOT NULL DEFAULT 'whatsapp';
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'open';
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS assigned_user_id UUID NULL;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS last_message_at TIMESTAMPTZ;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS unread_count INT NOT NULL DEFAULT 0;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb;
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
+            ALTER TABLE comms_threads ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
             CREATE INDEX IF NOT EXISTS idx_comms_threads_phone_provider ON comms_threads(provider, phone_e164);
             CREATE INDEX IF NOT EXISTS idx_comms_threads_person ON comms_threads(person_id) WHERE person_id IS NOT NULL;
             CREATE INDEX IF NOT EXISTS idx_comms_threads_status ON comms_threads(status);
@@ -203,6 +249,19 @@ async def _ensure_schema(pool) -> None:
                 raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb,
                 created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
             );
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'mock';
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS external_message_id TEXT;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS direction TEXT NOT NULL DEFAULT 'system';
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS message_type TEXT NOT NULL DEFAULT 'text';
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS body TEXT NOT NULL DEFAULT '';
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS media_url TEXT;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS media_mime_type TEXT;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS delivery_status TEXT NOT NULL DEFAULT 'pending';
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS sent_at TIMESTAMPTZ;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS delivered_at TIMESTAMPTZ;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS read_at TIMESTAMPTZ;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb;
+            ALTER TABLE comms_messages ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
             CREATE INDEX IF NOT EXISTS idx_comms_messages_thread ON comms_messages(thread_id, created_at DESC);
             CREATE INDEX IF NOT EXISTS idx_comms_messages_external ON comms_messages(external_message_id) WHERE external_message_id IS NOT NULL;
             CREATE TABLE IF NOT EXISTS comms_call_logs (
@@ -223,6 +282,21 @@ async def _ensure_schema(pool) -> None:
                 raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb,
                 created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
             );
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS thread_id UUID NULL REFERENCES comms_threads(thread_id) ON DELETE SET NULL;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS person_id UUID NULL REFERENCES crm_people(person_id) ON DELETE SET NULL;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'mock';
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS external_call_id TEXT;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS phone_e164 TEXT NOT NULL DEFAULT '';
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS direction TEXT NOT NULL DEFAULT 'inbound';
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'completed';
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS ended_at TIMESTAMPTZ;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS duration_seconds INT;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS recording_url TEXT;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS transcript_id UUID;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS transcript_text TEXT;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb;
+            ALTER TABLE comms_call_logs ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
             CREATE INDEX IF NOT EXISTS idx_comms_call_logs_phone ON comms_call_logs(phone_e164);
             CREATE INDEX IF NOT EXISTS idx_comms_call_logs_thread ON comms_call_logs(thread_id) WHERE thread_id IS NOT NULL;
             CREATE TABLE IF NOT EXISTS comms_settings (
@@ -422,6 +496,54 @@ async def list_messages(
     return {"messages": messages, "thread": await get_thread(thread_id, request)}
 
 
+@router.get("/threads/{thread_id}/calls")
+async def list_thread_calls(
+    thread_id: str,
+    request: Request,
+    limit: int = 25,
+    offset: int = 0,
+    _: UserPrincipal = Depends(get_current_user),
+):
+    pool = await _pool(request)
+    limit = max(1, min(limit, 100))
+    offset = max(0, offset)
+    async with pool.acquire() as conn:
+        rows = await conn.fetch(
+            """
+            SELECT *
+            FROM comms_call_logs
+            WHERE thread_id = $1::uuid
+            ORDER BY started_at DESC, created_at DESC
+            LIMIT $2 OFFSET $3
+            """,
+            thread_id,
+            limit,
+            offset,
+        )
+    calls = [
+        {
+            "callId": str(row["call_id"]),
+            "threadId": str(row["thread_id"]) if row["thread_id"] else None,
+            "personId": str(row["person_id"]) if row["person_id"] else None,
+            "provider": row["provider"],
+            "externalCallId": row["external_call_id"],
+            "phoneE164": row["phone_e164"],
+            "direction": row["direction"],
+            "status": row["status"],
+            "startedAt": row["started_at"].isoformat(),
+            "endedAt": row["ended_at"].isoformat() if row["ended_at"] else None,
+            "durationSeconds": row["duration_seconds"],
+            "recordingUrl": row["recording_url"],
+            "transcriptId": str(row["transcript_id"]) if row["transcript_id"] else None,
+            "transcriptText": row["transcript_text"],
+            "rawPayload": _json_obj(row["raw_payload"]),
+            "createdAt": row["created_at"].isoformat(),
+        }
+        for row in rows
+    ]
+    return {"calls": calls, "thread": await get_thread(thread_id, request)}
+
+
 @router.post("/threads/{thread_id}/messages")
 async def send_message(
     thread_id: str,
@@ -465,11 +587,15 @@ async def link_person(
     thread_id: str,
     body: LinkPersonBody,
     request: Request,
-    _: UserPrincipal = Depends(get_current_user),
+    user: UserPrincipal = Depends(get_current_user),
 ):
     pool = await _pool(request)
     async with pool.acquire() as conn:
-        exists = await conn.fetchval("SELECT EXISTS (SELECT 1 FROM crm_people WHERE person_id = $1::uuid)", body.personId)
+        exists = await conn.fetchval(
+            "SELECT EXISTS (SELECT 1 FROM crm_people WHERE person_id = $1::uuid AND tenant_id = $2)",
+            body.personId,
+            user.tenant_id,
+        )
         if not exists:
             raise HTTPException(status_code=404, detail="CRM person not found")
         updated = await conn.execute(
@@ -481,36 +607,127 @@ async def link_person(
 
 
 @router.post("/threads/{thread_id}/notes")
-async def add_note(thread_id: str, body: NoteBody, request: Request, _: UserPrincipal = Depends(get_current_user)):
+async def add_note(thread_id: str, body: NoteBody, request: Request, user: UserPrincipal = Depends(get_current_user)):
     pool = await _pool(request)
     async with pool.acquire() as conn:
-        msg_id = await conn.fetchval(
-            """
-            INSERT INTO comms_messages (thread_id, provider, direction, message_type, body, delivery_status)
-            VALUES ($1::uuid, 'system', 'system', 'text', $2, 'delivered')
-            RETURNING message_id
-            """,
-            thread_id,
-            f"Note: {body.content}",
-        )
-    return {"messageId": str(msg_id)}
+        async with conn.transaction():
+            thread, lead_id = await _thread_context(conn, thread_id, user.tenant_id)
+            msg_id = await conn.fetchval(
+                """
+                INSERT INTO comms_messages (thread_id, provider, direction, message_type, body, delivery_status)
+                VALUES ($1::uuid, 'system', 'system', 'text', $2, 'delivered')
+                RETURNING message_id
+                """,
+                thread_id,
+                f"Note: {body.content}",
+            )
+            interaction_id = None
+            canonical_message_id = None
+            if thread["person_id"]:
+                interaction_id = await conn.fetchval(
+                    """
+                    INSERT INTO intel_interactions (
+                        interaction_id, tenant_id, person_id, lead_id, channel,
+                        interaction_type, happened_at, summary, source_ref, metadata_json
+                    ) VALUES (
+                        gen_random_uuid(), COALESCE($1, 'tenant_velocity'), $2::uuid, $3::uuid, 'whatsapp',
+                        'operator_note', NOW(), $4, $5, $6::jsonb
+                    )
+                    RETURNING interaction_id
+                    """,
+                    user.tenant_id,
+                    thread["person_id"],
+                    lead_id,
+                    body.content,
+                    f"comms:{thread_id}",
+                    json.dumps({"source": "comms_thread_note", "thread_id": thread_id, "message_id": str(msg_id)}),
+                )
+                canonical_message_id = await conn.fetchval(
+                    """
+                    INSERT INTO intel_messages (
+                        message_id, interaction_id, thread_id, sender_role, sender_name,
+                        message_text, delivered_at, metadata_json
+                    ) VALUES (
+                        gen_random_uuid(), $1::uuid, $2::uuid, 'operator', 'iPad operator',
+                        $3, NOW(), $4::jsonb
+                    )
+                    RETURNING message_id
+                    """,
+                    interaction_id,
+                    thread_id,
+                    body.content,
+                    json.dumps({"source": "comms_thread_note"}),
+                )
+    return {
+        "messageId": str(msg_id),
+        "canonicalInteractionId": str(interaction_id) if interaction_id else None,
+        "canonicalMessageId": str(canonical_message_id) if canonical_message_id else None,
+    }
 
 
 @router.post("/threads/{thread_id}/tasks")
-async def add_task(thread_id: str, body: TaskBody, request: Request, _: UserPrincipal = Depends(get_current_user)):
+async def add_task(thread_id: str, body: TaskBody, request: Request, user: UserPrincipal = Depends(get_current_user)):
     pool = await _pool(request)
     text = f"Task: {body.title}" + (f" (Due: {body.dueAt})" if body.dueAt else "")
     async with pool.acquire() as conn:
-        msg_id = await conn.fetchval(
-            """
-            INSERT INTO comms_messages (thread_id, provider, direction, message_type, body, delivery_status)
-            VALUES ($1::uuid, 'system', 'system', 'text', $2, 'delivered')
-            RETURNING message_id
-            """,
-            thread_id,
-            text,
-        )
-    return {"messageId": str(msg_id)}
+        async with conn.transaction():
+            thread, lead_id = await _thread_context(conn, thread_id, user.tenant_id)
+            msg_id = await conn.fetchval(
+                """
+                INSERT INTO comms_messages (thread_id, provider, direction, message_type, body, delivery_status)
+                VALUES ($1::uuid, 'system', 'system', 'text', $2, 'delivered')
+                RETURNING message_id
+                """,
+                thread_id,
+                text,
+            )
+            reminder_id = None
+            interaction_id = None
+            if thread["person_id"]:
+                interaction_id = await conn.fetchval(
+                    """
+                    INSERT INTO intel_interactions (
+                        interaction_id, tenant_id, person_id, lead_id, channel,
+                        interaction_type, happened_at, summary, source_ref, metadata_json
+                    ) VALUES (
+                        gen_random_uuid(), COALESCE($1, 'tenant_velocity'), $2::uuid, $3::uuid, 'whatsapp',
+                        'next_best_action', NOW(), $4, $5, $6::jsonb
+                    )
+                    RETURNING interaction_id
+                    """,
+                    user.tenant_id,
+                    thread["person_id"],
+                    lead_id,
+                    body.title,
+                    f"comms:{thread_id}",
+                    json.dumps({"source": "comms_thread_task", "thread_id": thread_id, "message_id": str(msg_id)}),
+                )
+                reminder_id = await conn.fetchval(
+                    """
+                    INSERT INTO intel_reminders (
+                        reminder_id, tenant_id, person_id, lead_id, interaction_id,
+                        reminder_type, title, notes, due_at, status, priority,
+                        created_by_type, created_at
+                    ) VALUES (
+                        gen_random_uuid(), COALESCE($1, 'tenant_velocity'), $2::uuid, $3::uuid, $4::uuid,
+                        'follow_up', $5, $6, $7, 'pending', $8, 'human', NOW()
+                    )
+                    RETURNING reminder_id
+                    """,
+                    user.tenant_id,
+                    thread["person_id"],
+                    lead_id,
+                    interaction_id,
+                    body.title,
+                    body.notes,
+                    _optional_datetime(body.dueAt),
+                    body.priority,
+                )
+    return {
+        "messageId": str(msg_id),
+        "canonicalInteractionId": str(interaction_id) if interaction_id else None,
+        "canonicalReminderId": str(reminder_id) if reminder_id else None,
+    }
 
 
 @router.post("/webhooks/{provider}")
@@ -572,17 +789,53 @@ async def test_provider(request: Request, _: UserPrincipal = Depends(get_current
 @router.post("/recordings/transcribe")
 async def transcribe_recording(body: TranscribeBody, request: Request, _: UserPrincipal = Depends(get_current_user)):
     pool = await _pool(request)
+    config = await _load_config(pool)
+    configured_provider = str(config.get("transcription_provider") or "").strip().lower()
+    env_provider = os.getenv("COMMS_TRANSCRIPTION_PROVIDER", "none").strip().lower()
+    provider = env_provider if configured_provider in {"", "none", "disabled"} else configured_provider
+    recording_url = body.recordingUrl
+    if body.callId and not recording_url:
+        async with pool.acquire() as conn:
+            recording_url = await conn.fetchval(
+                "SELECT recording_url FROM comms_call_logs WHERE call_id = $1::uuid",
+                body.callId,
+            )
+    if not recording_url:
+        raise HTTPException(status_code=422, detail="recordingUrl is required when callId has no stored recording_url.")
+
+    try:
+        result = await run_transcription(recording_url, provider=provider)
+    except TranscriptionError as exc:
+        if body.callId:
+            async with pool.acquire() as conn:
+                await conn.execute(
+                    "UPDATE comms_call_logs SET transcript_text = $1 WHERE call_id = $2::uuid",
+                    f"Transcription failed: {exc}",
+                    body.callId,
+                )
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+
     if body.callId:
         async with pool.acquire() as conn:
             await conn.execute(
-                "UPDATE comms_call_logs SET transcript_text = $1 WHERE call_id = $2::uuid",
-                "Transcription pending. Configure COMMS_TRANSCRIPTION_PROVIDER to enable processing.",
+                """
+                UPDATE comms_call_logs
+                SET transcript_text = $1,
+                    raw_payload = COALESCE(raw_payload, '{}'::jsonb) || $2::jsonb
+                WHERE call_id = $3::uuid
+                """,
+                result["text"],
+                json.dumps({"transcription": {"provider": result["provider"], "language": result["language"]}}),
                 body.callId,
             )
     return {
         "success": True,
-        "status": "pending",
-        "message": "Transcription intake recorded. A real transcription worker/provider is still required.",
+        "status": "completed",
+        "message": "Transcription completed.",
         "callId": body.callId,
         "recordingUrl": body.recordingUrl,
+        "provider": result["provider"],
+        "language": result["language"],
+        "text": result["text"],
+        "segments": result["segments"],
     }