sagnik/Project_Velocity
2
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: CRM v2, Richer synthetic data, Canvas JSON Components

Browse Source
This commit is contained in:
Sagnik
2026-04-23 22:00:44 +05:30
parent 6cdc366718
commit f04571bd7b
54 changed files with 89916 additions and 578 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
backend/api/routes_crm_imports.py
View File

@@ -97,6 +97,18 @@ class CreateReminderRequest(BaseModel):
priority: str = "normal"
class ClientDataPatchRequest(BaseModel):
full_name: str | None = Field(default=None, max_length=256)
primary_email: str | None = Field(default=None, max_length=256)
primary_phone: str | None = Field(default=None, max_length=64)
buyer_type: str | None = Field(default=None, max_length=128)
communication_preference: str | None = Field(default=None, max_length=64)
best_contact_time: str | None = Field(default=None, max_length=128)
lead_status: str | None = Field(default=None, max_length=64)
budget_band: str | None = Field(default=None, max_length=128)
urgency: str | None = Field(default=None, max_length=64)
# ── Import Endpoints ──────────────────────────────────────────────────────────
@router.post("/crm/imports", status_code=201, tags=["CRM Imports"])
@@ -796,3 +808,172 @@ async def get_qd_score(request: Request, person_id: str) -> dict[str, Any]:
],
},
}
# ── Oracle Client Data Lens ───────────────────────────────────────────────────
@router.get("/crm/client-data", tags=["CRM Client Data"])
async def list_client_data(
request: Request,
search: str | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
pool = await _get_pool(request)
params: list[Any] = []
where = "1=1"
if search:
params.append(f"%{search.lower()}%")
where = f"(lower(p.full_name) LIKE ${len(params)} OR lower(COALESCE(p.primary_phone,'')) LIKE ${len(params)} OR lower(COALESCE(p.primary_email,'')) LIKE ${len(params)} OR lower(COALESCE(pi.projects,'')) LIKE ${len(params)})"
params.extend([limit, offset])
async with pool.acquire() as conn:
rows = await conn.fetch(
f"""
WITH interests AS (
SELECT person_id, string_agg(DISTINCT project_name, ', ') AS projects, COUNT(*)::int AS interest_count
FROM crm_property_interests GROUP BY person_id
),
qd AS (
SELECT DISTINCT ON (person_id) person_id, current_value
FROM intel_qd_scores
ORDER BY person_id, current_value DESC, computed_at DESC
)
SELECT p.person_id::text, p.full_name, p.primary_email, p.primary_phone, p.buyer_type,
p.broker_name, p.communication_preference, p.best_contact_time,
l.lead_id::text, l.status::text AS lead_status, l.budget_band, l.urgency,
COALESCE(qd.current_value, p.engagement_score, 0)::float AS qd_score,
lc.last_contact_at, lc.last_channel, lc.days_since_contact,
nba.recommended_action AS next_best_action, nba.priority AS next_action_priority,
COALESCE(pi.projects, '') AS projects, COALESCE(pi.interest_count, 0)::int AS interest_count
FROM crm_people p
LEFT JOIN LATERAL (
SELECT * FROM crm_leads l WHERE l.person_id = p.person_id ORDER BY l.updated_at DESC LIMIT 1
) l ON TRUE
LEFT JOIN interests pi ON pi.person_id = p.person_id
LEFT JOIN qd ON qd.person_id = p.person_id
LEFT JOIN read_last_contacted lc ON lc.person_id = p.person_id
LEFT JOIN read_next_best_action nba ON nba.person_id = p.person_id
WHERE {where}
ORDER BY lc.last_contact_at DESC NULLS LAST, qd_score DESC, p.full_name ASC
LIMIT ${len(params)-1} OFFSET ${len(params)}
""",
*params,
)
return {"status": "ok", "data": [dict(r) for r in rows], "meta": {"count": len(rows), "limit": limit, "offset": offset}}
@router.get("/crm/client-data/{person_id}", tags=["CRM Client Data"])
async def get_client_data(request: Request, person_id: str) -> dict[str, Any]:
pool = await _get_pool(request)
async with pool.acquire() as conn:
base = await conn.fetchrow(
"""
SELECT p.*, l.lead_id::text, l.status::text AS lead_status, l.budget_band, l.urgency,
l.financing_posture, l.timeline_to_decision, l.broker_team,
lc.last_contact_at, lc.last_channel, lc.days_since_contact,
nba.recommended_action, nba.priority AS next_action_priority, nba.rationale,
nba.suggested_channel, nba.due_within_days
FROM crm_people p
LEFT JOIN LATERAL (
SELECT * FROM crm_leads l WHERE l.person_id = p.person_id ORDER BY l.updated_at DESC LIMIT 1
) l ON TRUE
LEFT JOIN read_last_contacted lc ON lc.person_id = p.person_id
LEFT JOIN read_next_best_action nba ON nba.person_id = p.person_id
WHERE p.person_id = $1::uuid
""",
person_id,
)
if not base:
raise HTTPException(status_code=404, detail=f"Client '{person_id}' not found.")
interests = await conn.fetch("SELECT * FROM crm_property_interests WHERE person_id = $1::uuid ORDER BY priority ASC, created_at DESC", person_id)
opportunities = await conn.fetch(
"""
SELECT o.*, ip.project_name FROM crm_opportunities o
JOIN crm_leads l ON l.lead_id = o.lead_id
LEFT JOIN inventory_projects ip ON ip.project_id = o.project_id
WHERE l.person_id = $1::uuid ORDER BY o.updated_at DESC
""",
person_id,
)
facts = await conn.fetch("SELECT * FROM intel_extracted_facts WHERE person_id = $1::uuid ORDER BY extracted_at DESC LIMIT 50", person_id)
qd = await conn.fetch("SELECT * FROM intel_qd_scores WHERE person_id = $1::uuid ORDER BY current_value DESC", person_id)
timeline = await _client_timeline(conn, person_id, 60)
return {
"status": "ok",
"data": {
"profile": dict(base),
"property_interests": [dict(r) for r in interests],
"opportunities": [dict(r) for r in opportunities],
"extracted_facts": [dict(r) for r in facts],
"qd_scores": [dict(r) for r in qd],
"timeline": timeline,
},
}
@router.patch("/crm/client-data/{person_id}", tags=["CRM Client Data"])
async def patch_client_data(request: Request, person_id: str, body: ClientDataPatchRequest) -> dict[str, Any]:
pool = await _get_pool(request)
payload = body.model_dump(exclude_unset=True)
person_fields = {k: payload[k] for k in ("full_name", "primary_email", "primary_phone", "buyer_type", "communication_preference", "best_contact_time") if k in payload}
lead_fields = {k: payload[k] for k in ("budget_band", "urgency") if k in payload}
async with pool.acquire() as conn:
async with conn.transaction():
if person_fields:
sets = ", ".join(f"{key} = ${idx}" for idx, key in enumerate(person_fields, start=1))
await conn.execute(f"UPDATE crm_people SET {sets}, updated_at = NOW() WHERE person_id = ${len(person_fields)+1}::uuid", *person_fields.values(), person_id)
if lead_fields:
lead_id = await conn.fetchval("SELECT lead_id FROM crm_leads WHERE person_id = $1::uuid ORDER BY updated_at DESC LIMIT 1", person_id)
if lead_id:
sets = ", ".join(f"{key} = ${idx}" for idx, key in enumerate(lead_fields, start=1))
await conn.execute(f"UPDATE crm_leads SET {sets}, updated_at = NOW() WHERE lead_id = ${len(lead_fields)+1}::uuid", *lead_fields.values(), str(lead_id))
if "lead_status" in payload:
await conn.execute(
"UPDATE crm_leads SET status = $1::crm_lead_status, updated_at = NOW() WHERE person_id = $2::uuid",
payload["lead_status"],
person_id,
)
return {"status": "ok", "data": {"person_id": person_id, "updated": sorted(payload.keys())}}
@router.get("/crm/client-data/{person_id}/timeline", tags=["CRM Client Data"])
async def get_client_data_timeline(request: Request, person_id: str, limit: int = Query(default=80, ge=1, le=200)) -> dict[str, Any]:
pool = await _get_pool(request)
async with pool.acquire() as conn:
rows = await _client_timeline(conn, person_id, limit)
return {"status": "ok", "data": rows}
@router.post("/crm/client-data/{person_id}/tasks", status_code=201, tags=["CRM Client Data"])
async def create_client_data_task(request: Request, person_id: str, body: CreateReminderRequest) -> dict[str, Any]:
patched = body.model_copy(update={"person_id": person_id})
return await create_task(request, patched)
async def _client_timeline(conn: Any, person_id: str, limit: int) -> list[dict[str, Any]]:
rows = await conn.fetch(
"""
SELECT * FROM (
SELECT i.interaction_id::text AS id, i.channel::text AS type, i.interaction_type AS title,
i.summary, i.happened_at AS date, i.broker_name AS actor
FROM intel_interactions i WHERE i.person_id = $1::uuid
UNION ALL
SELECT m.message_id::text, 'message', m.sender_role, m.message_text, m.delivered_at, m.sender_name
FROM intel_messages m JOIN intel_interactions i ON i.interaction_id = m.interaction_id WHERE i.person_id = $1::uuid
UNION ALL
SELECT r.reminder_id::text, 'reminder', r.title, r.notes, r.due_at, r.priority
FROM intel_reminders r WHERE r.person_id = $1::uuid
UNION ALL
SELECT v.visit_id::text, 'visit', COALESCE(v.outcome_type, 'site_visit'), COALESCE(v.visit_notes, v.broker_notes), v.visited_at, v.broker_name
FROM intel_visits v WHERE v.person_id = $1::uuid
UNION ALL
SELECT q.timeseries_id::text, 'qd', q.score_type, q.signal_source, q.timestamp, q.value::text
FROM intel_qd_timeseries q WHERE q.person_id = $1::uuid
) events
ORDER BY date DESC NULLS LAST
LIMIT $2
""",
person_id,
limit,
)
return [dict(r) for r in rows]

40
backend/api/routes_oracle.py
View File

@@ -4,6 +4,7 @@ from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from backend.oracle.action_service import oracle_action_service
from backend.oracle.natural_db_agent import natural_db_agent
from backend.oracle.persona_service import persona_service
from backend.services.mcp_registry import mcp_registry
from backend.services.nemoclaw_runtime import nemoclaw_runtime
@@ -33,6 +34,12 @@ class OracleWritebackRequest(BaseModel):
writeback_payload: dict = Field(default_factory=dict)
class OracleQueryRequest(BaseModel):
prompt: str = Field(..., min_length=1, max_length=4096)
row_limit: int = Field(default=100, ge=1, le=500)
context: dict = Field(default_factory=dict)
@router.get("/health")
async def oracle_health() -> dict:
return {
@@ -43,6 +50,39 @@ async def oracle_health() -> dict:
}
@router.get("/data-health")
async def oracle_data_health(request: Request) -> dict:
pool = getattr(request.app.state, "db_pool", None)
if pool is None:
raise HTTPException(status_code=503, detail="Database unavailable.")
async with pool.acquire() as conn:
data = await natural_db_agent.data_health(conn)
return {
"status": "ok",
"data": data,
}
@router.get("/schema-catalog")
async def oracle_schema_catalog(request: Request) -> dict:
pool = getattr(request.app.state, "db_pool", None)
if pool is None:
raise HTTPException(status_code=503, detail="Database unavailable.")
async with pool.acquire() as conn:
catalog = await natural_db_agent.schema_catalog(conn)
return {"status": "ok", "data": catalog}
@router.post("/query")
async def oracle_query(request: Request, payload: OracleQueryRequest) -> dict:
pool = getattr(request.app.state, "db_pool", None)
if pool is None:
raise HTTPException(status_code=503, detail="Database unavailable.")
async with pool.acquire() as conn:
result = await natural_db_agent.execute_prompt(payload.prompt, row_limit=payload.row_limit, conn=conn)
return {"status": "ok", "data": result.as_dict()}
@router.get("/mcp/tools")
async def oracle_mcp_tools() -> dict:
return {"status": "ok", "data": mcp_registry.list_tools()}

187
backend/db/schema_crm_canonical.sql
View File

@@ -220,9 +220,13 @@ CREATE TABLE IF NOT EXISTS crm_property_interests (
budget_max DECIMAL(15, 2),
priority INTEGER DEFAULT 1, -- 1 = primary, 2 = secondary
notes TEXT,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
ALTER TABLE crm_property_interests
ADD COLUMN IF NOT EXISTS metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb;
CREATE INDEX IF NOT EXISTS idx_crm_pi_person ON crm_property_interests (person_id);
CREATE INDEX IF NOT EXISTS idx_crm_pi_project ON crm_property_interests (project_id);
@@ -706,3 +710,186 @@ COMMENT ON TABLE intel_qd_scores IS 'Latest QD summary by score_type per client.
COMMENT ON TABLE inventory_projects IS 'Master project catalog. 14 canonical Kolkata projects seeded.';
COMMENT ON TABLE workflow_import_batches IS 'RawImportBatch contract. Immutable after upload.';
COMMENT ON TABLE workflow_writebacks IS 'AI-proposed canonical mutations. Never auto-execute without approval.';
-- -----------------------------------------------------------------------------
-- Synthetic CRM v2 enrichment columns and Oracle read models
-- -----------------------------------------------------------------------------
ALTER TABLE crm_people
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS engagement_score FLOAT,
ADD COLUMN IF NOT EXISTS communication_preference TEXT,
ADD COLUMN IF NOT EXISTS best_contact_time TEXT;
ALTER TABLE crm_households
ADD COLUMN IF NOT EXISTS size INTEGER,
ADD COLUMN IF NOT EXISTS combined_budget_band TEXT,
ADD COLUMN IF NOT EXISTS decision_maker_id UUID REFERENCES crm_people(person_id) ON DELETE SET NULL;
ALTER TABLE crm_leads
ADD COLUMN IF NOT EXISTS stage TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS broker_team TEXT,
ADD COLUMN IF NOT EXISTS engagement_score FLOAT,
ADD COLUMN IF NOT EXISTS last_activity_at TIMESTAMPTZ;
ALTER TABLE crm_opportunities
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS deal_velocity TEXT,
ADD COLUMN IF NOT EXISTS risk_factors JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE crm_property_interests
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS last_discussed_at TIMESTAMPTZ;
ALTER TABLE crm_stage_history
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS transition_duration_days INTEGER;
ALTER TABLE intel_interactions
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS broker_team TEXT,
ADD COLUMN IF NOT EXISTS sentiment TEXT,
ADD COLUMN IF NOT EXISTS sentiment_score FLOAT,
ADD COLUMN IF NOT EXISTS intent_label TEXT,
ADD COLUMN IF NOT EXISTS emotion_tags JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS client_engagement_level TEXT;
ALTER TABLE intel_calls
ADD COLUMN IF NOT EXISTS objection_tags JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS outcome_summary TEXT,
ADD COLUMN IF NOT EXISTS follow_up_actions JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS call_quality_score FLOAT;
ALTER TABLE intel_emails
ADD COLUMN IF NOT EXISTS sentiment TEXT,
ADD COLUMN IF NOT EXISTS intent_label TEXT,
ADD COLUMN IF NOT EXISTS engagement_score FLOAT,
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS response_expected BOOLEAN;
ALTER TABLE intel_reminders
ADD COLUMN IF NOT EXISTS interaction_id UUID REFERENCES intel_interactions(interaction_id) ON DELETE SET NULL,
ADD COLUMN IF NOT EXISTS context_snippet TEXT,
ADD COLUMN IF NOT EXISTS completion_percentage INTEGER,
ADD COLUMN IF NOT EXISTS overdue_days INTEGER,
ADD COLUMN IF NOT EXISTS outcome_notes TEXT;
ALTER TABLE intel_transcripts
ADD COLUMN IF NOT EXISTS call_outcome TEXT,
ADD COLUMN IF NOT EXISTS follow_up_required BOOLEAN,
ADD COLUMN IF NOT EXISTS emotion_tags JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS call_summary TEXT;
ALTER TABLE intel_visits
ADD COLUMN IF NOT EXISTS outcome_type TEXT,
ADD COLUMN IF NOT EXISTS visit_duration_minutes INTEGER,
ADD COLUMN IF NOT EXISTS interest_signals JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS interest_score FLOAT,
ADD COLUMN IF NOT EXISTS companion_type TEXT,
ADD COLUMN IF NOT EXISTS companion_count INTEGER,
ADD COLUMN IF NOT EXISTS objections_raised JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS follow_up_required BOOLEAN,
ADD COLUMN IF NOT EXISTS next_steps TEXT,
ADD COLUMN IF NOT EXISTS broker_notes TEXT,
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT;
ALTER TABLE intel_whatsapp_threads
ADD COLUMN IF NOT EXISTS broker_id TEXT,
ADD COLUMN IF NOT EXISTS broker_name TEXT,
ADD COLUMN IF NOT EXISTS topic_category TEXT,
ADD COLUMN IF NOT EXISTS sentiment_direction TEXT,
ADD COLUMN IF NOT EXISTS resolution_status TEXT;
ALTER TABLE intel_qd_scores
ADD COLUMN IF NOT EXISTS score_drivers JSONB NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS trend_direction TEXT,
ADD COLUMN IF NOT EXISTS explanation TEXT,
ADD COLUMN IF NOT EXISTS confidence FLOAT;
ALTER TABLE intel_qd_timeseries
ADD COLUMN IF NOT EXISTS broker_id TEXT;
CREATE TABLE IF NOT EXISTS intel_email_threads (
thread_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
subject TEXT,
first_email_at TIMESTAMPTZ,
last_email_at TIMESTAMPTZ,
email_count INTEGER DEFAULT 0,
participants JSONB NOT NULL DEFAULT '[]'::jsonb,
status TEXT,
broker_id TEXT,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb
);
CREATE TABLE IF NOT EXISTS intel_call_objections (
objection_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
call_id UUID REFERENCES intel_calls(call_id) ON DELETE CASCADE,
objection_type TEXT,
category TEXT,
severity TEXT,
status TEXT,
client_quote TEXT,
agent_response TEXT,
resolution_strategy TEXT,
extracted_at TIMESTAMPTZ,
confidence_score FLOAT,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb
);
CREATE TABLE IF NOT EXISTS intel_extracted_facts (
fact_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
interaction_id UUID REFERENCES intel_interactions(interaction_id) ON DELETE SET NULL,
person_id UUID REFERENCES crm_people(person_id) ON DELETE CASCADE,
fact_type TEXT NOT NULL,
fact_value TEXT,
confidence FLOAT,
extracted_from TEXT,
source_context TEXT,
extracted_at TIMESTAMPTZ,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb
);
CREATE TABLE IF NOT EXISTS read_last_contacted (
person_id UUID PRIMARY KEY REFERENCES crm_people(person_id) ON DELETE CASCADE,
last_contact_at TIMESTAMPTZ,
last_channel TEXT,
last_interaction_type TEXT,
days_since_contact INTEGER,
interactions_last_7d INTEGER,
interactions_last_30d INTEGER,
interactions_last_90d INTEGER,
total_interactions INTEGER,
current_stage TEXT,
broker_id TEXT,
broker_name TEXT,
computed_at TIMESTAMPTZ,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb
);
CREATE TABLE IF NOT EXISTS read_next_best_action (
person_id UUID PRIMARY KEY REFERENCES crm_people(person_id) ON DELETE CASCADE,
recommended_action TEXT,
priority TEXT,
rationale TEXT,
suggested_channel TEXT,
due_within_days INTEGER,
broker_id TEXT,
broker_name TEXT,
opportunity_context TEXT,
computed_at TIMESTAMPTZ,
metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb
);
CREATE INDEX IF NOT EXISTS idx_read_last_contacted_at ON read_last_contacted (last_contact_at DESC);
CREATE INDEX IF NOT EXISTS idx_read_next_best_priority ON read_next_best_action (priority);
CREATE INDEX IF NOT EXISTS idx_intel_facts_person ON intel_extracted_facts (person_id, extracted_at DESC);
CREATE INDEX IF NOT EXISTS idx_intel_objections_call ON intel_call_objections (call_id);

10
backend/oracle/canvas_service.py
View File

@@ -98,6 +98,7 @@ def _json_safe(value: Any) -> Any:
def _normalize_component(component: dict[str, Any]) -> dict[str, Any]:
normalized = deepcopy(component)
normalized["componentId"] = _stringify(normalized.get("componentId"))
normalized["dataRows"] = _json_array(normalized.get("dataRows"))
descriptor = _json_object(normalized.get("dataSourceDescriptor"))
if descriptor.get("descriptorId") is not None:
descriptor["descriptorId"] = _stringify(descriptor["descriptorId"])
@@ -126,6 +127,7 @@ def _deserialize_component_row(row: Any) -> dict[str, Any]:
"version": row["version"],
"lifecycleState": row["lifecycle_state"],
"dataSourceDescriptor": row["data_source_descriptor"],
"dataRows": row["data_rows"],
"visualizationParameters": row["visualization_parameters"],
"dataBindings": row["data_bindings"],
"provenance": row["provenance"],
@@ -602,13 +604,13 @@ class CanvasService:
"""
INSERT INTO oracle_canvas_components (
component_id, page_id, tenant_id, type, title, description, version, lifecycle_state,
data_source_descriptor, visualization_parameters, data_bindings, provenance,
data_source_descriptor, data_rows, visualization_parameters, data_bindings, provenance,
rendering_hints, layout, access_controls, style_signature, validation_state, audit_log
)
VALUES (
$1::uuid, $2::uuid, $3, $4, $5, $6, $7, $8,
$9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb,
$13::jsonb, $14::jsonb, $15::jsonb, $16::jsonb, $17::jsonb, $18::text[]
$9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13::jsonb,
$14::jsonb, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::text[]
)
ON CONFLICT (component_id)
DO UPDATE SET
@@ -617,6 +619,7 @@ class CanvasService:
version = EXCLUDED.version,
lifecycle_state = EXCLUDED.lifecycle_state,
data_source_descriptor = EXCLUDED.data_source_descriptor,
data_rows = EXCLUDED.data_rows,
visualization_parameters = EXCLUDED.visualization_parameters,
data_bindings = EXCLUDED.data_bindings,
provenance = EXCLUDED.provenance,
@@ -637,6 +640,7 @@ class CanvasService:
int(component.get("version", 1)),
component.get("lifecycleState", "active"),
json.dumps(_json_safe(component.get("dataSourceDescriptor", {}))),
json.dumps(_json_safe(component.get("dataRows", []))),
json.dumps(_json_safe(component.get("visualizationParameters", {}))),
json.dumps(_json_safe(component.get("dataBindings", {}))),
json.dumps(_json_safe(component.get("provenance", {}))),

412
backend/oracle/data_access_gateway.py
View File

@@ -2,8 +2,8 @@
oracle/data_access_gateway.py
Read-only, policy-aware PostgreSQL query executor for Oracle datasets.
Nemoclaw is treated strictly as a planner. The gateway executes only
whitelisted dataset queries and always injects the actor's tenant scope.
Nemoclaw/LLM is treated strictly as a planner. The gateway executes only
whitelisted read models and always applies policy before touching data.
"""
from __future__ import annotations
@@ -21,9 +21,15 @@ from .policy_service import PolicyContext, PolicyService
logger = logging.getLogger(__name__)
_DB_URL = os.getenv("DATABASE_URL", "")
_ALLOW_IN_MEMORY = os.getenv("ORACLE_ALLOW_IN_MEMORY_FALLBACK", "").lower() in {"1", "true", "yes"}
_DATASET_ALIASES = {
"crm_last_interacted_clients": "oracle_last_contacted_clients",
"crm_top_interested_clients": "oracle_top_interested_clients",
"crm_interaction_timeline": "oracle_client_interaction_timeline",
"crm_property_interest_rollup": "oracle_property_interest_rollup",
}
@dataclass
class QueryExecutionResult:
@@ -32,7 +38,29 @@ class QueryExecutionResult:
def _db_ready() -> bool:
return bool(_DB_URL and not _DB_URL.startswith("PLACEHOLDER") and asyncpg is not None)
if asyncpg is None:
return False
database_url = os.getenv("DATABASE_URL", "")
if database_url and not database_url.startswith("PLACEHOLDER"):
return True
return all(
os.getenv(name)
for name in ("VELOCITY_DB_NAME", "VELOCITY_DB_USER", "VELOCITY_DB_PASSWORD")
)
async def _connect_db() -> Any:
assert asyncpg is not None
database_url = os.getenv("DATABASE_URL", "")
if database_url and not database_url.startswith("PLACEHOLDER"):
return await asyncpg.connect(database_url)
return await asyncpg.connect(
host=os.getenv("VELOCITY_DB_HOST", "localhost"),
port=int(os.getenv("VELOCITY_DB_PORT", "5432")),
database=os.environ["VELOCITY_DB_NAME"],
user=os.environ["VELOCITY_DB_USER"],
password=os.environ["VELOCITY_DB_PASSWORD"],
)
class DataAccessGateway:
@@ -61,7 +89,7 @@ class DataAccessGateway:
try:
rows = await self._query_dataset(
dataset=dataset,
dataset=_DATASET_ALIASES.get(dataset, dataset),
row_limit=validation.effective_row_limit,
ctx=ctx,
prompt=prompt,
@@ -82,8 +110,7 @@ class DataAccessGateway:
prompt: str,
) -> list[dict[str, Any]]:
sql, params = self._build_whitelisted_query(dataset, row_limit, ctx, prompt)
assert asyncpg is not None
conn = await asyncpg.connect(_DB_URL)
conn = await _connect_db()
try:
records = await conn.fetch(sql, *params)
finally:
@@ -101,23 +128,9 @@ class DataAccessGateway:
if dataset == "deals":
sql = """
SELECT
stage,
COUNT(*)::int AS count,
COALESCE(SUM(value), 0)::float AS value,
COALESCE(
json_agg(
json_build_object(
'id', lead_id,
'name', lead_name,
'company', company,
'value', value_label,
'avatar', avatar_url
)
ORDER BY value DESC NULLS LAST
) FILTER (WHERE lead_id IS NOT NULL),
'[]'::json
) AS leads
SELECT stage, COUNT(*)::int AS count, COALESCE(SUM(value), 0)::float AS value,
COALESCE(json_agg(json_build_object('id', lead_id, 'name', lead_name, 'company', company, 'value', value_label, 'avatar', avatar_url)
ORDER BY value DESC NULLS LAST) FILTER (WHERE lead_id IS NOT NULL), '[]'::json) AS leads
FROM deals
WHERE tenant_id = $1
GROUP BY stage
@@ -128,9 +141,7 @@ class DataAccessGateway:
if dataset == "lead_daily_snapshot":
sql = """
SELECT
source,
COALESCE(SUM(qd_weighted_score), 0)::float AS qd_weighted_volume
SELECT source, COALESCE(SUM(qd_weighted_score), 0)::float AS qd_weighted_volume
FROM lead_daily_snapshot
WHERE tenant_id = $1
GROUP BY source
@@ -141,14 +152,9 @@ class DataAccessGateway:
if dataset == "lead_geo_interest_rollup":
sql = """
SELECT
district,
lat,
lng,
COALESCE(lead_count, 0)::int AS lead_count,
COALESCE(avg_qd_score, 0)::float AS avg_qd_score,
COALESCE(x, 0)::float AS x,
COALESCE(y, 0)::float AS y
SELECT district, lat, lng, COALESCE(lead_count, 0)::int AS lead_count,
COALESCE(avg_qd_score, 0)::float AS avg_qd_score,
COALESCE(x, 0)::float AS x, COALESCE(y, 0)::float AS y
FROM lead_geo_interest_rollup
WHERE tenant_id = $1
ORDER BY lead_count DESC, district ASC
@@ -158,14 +164,11 @@ class DataAccessGateway:
if dataset == "broker_performance":
sql = """
SELECT
ROW_NUMBER() OVER (
ORDER BY COUNT(DISTINCT l.person_id) DESC, COALESCE(u.full_name, u.email, u.id::text) ASC
)::int AS rank,
COALESCE(u.full_name, u.email, u.id::text) AS name,
COUNT(DISTINCT l.person_id)::int AS deals_closed,
COALESCE(SUM(o.value), 0)::float AS revenue_generated,
u.avatar_url AS avatar
SELECT ROW_NUMBER() OVER (ORDER BY COUNT(DISTINCT l.person_id) DESC, COALESCE(u.full_name, u.email, u.id::text) ASC)::int AS rank,
COALESCE(u.full_name, u.email, u.id::text) AS name,
COUNT(DISTINCT l.person_id)::int AS deals_closed,
COALESCE(SUM(o.value), 0)::float AS revenue_generated,
u.avatar_url AS avatar
FROM users_and_roles u
LEFT JOIN crm_leads l ON l.assigned_user_id = u.id
LEFT JOIN crm_opportunities o ON o.lead_id = l.lead_id
@@ -173,16 +176,14 @@ class DataAccessGateway:
GROUP BY u.id, u.full_name, u.email, u.avatar_url
HAVING COUNT(DISTINCT l.person_id) > 0 OR COALESCE(SUM(o.value), 0) > 0
ORDER BY revenue_generated DESC, name ASC
LIMIT $2
LIMIT $1
"""
return sql, [ctx.tenant_id, row_limit]
return sql, [row_limit]
if dataset == "inventory_absorption":
sql = """
SELECT
period_label AS period,
COALESCE(absorption_rate, 0)::float AS absorption_rate,
COALESCE(target_rate, 0)::float AS target_rate
SELECT period_label AS period, COALESCE(absorption_rate, 0)::float AS absorption_rate,
COALESCE(target_rate, 0)::float AS target_rate
FROM inventory_absorption
WHERE tenant_id = $1
ORDER BY period_start ASC
@@ -196,16 +197,10 @@ class DataAccessGateway:
metric_name = "total_pipeline_value"
elif "quota" in lower_prompt or "attainment" in lower_prompt:
metric_name = "quota_attainment"
sql = """
SELECT
metric_value,
metric_label,
trend_value,
comparison_label
SELECT metric_value, metric_label, trend_value, comparison_label
FROM oracle_aggregated_metric
WHERE tenant_id = $1
AND metric_name = $2
WHERE tenant_id = $1 AND metric_name = $2
ORDER BY observed_at DESC
LIMIT 1
"""
@@ -214,27 +209,19 @@ class DataAccessGateway:
if dataset == "lead_activity_log":
if "follow-up" in lower_prompt or "queue" in lower_prompt:
sql = """
SELECT
lead_name AS name,
assigned_broker,
COALESCE(last_contact_hours_ago, 0)::int AS last_contact_hours_ago,
COALESCE(qd_score, 0)::float AS qd_score,
urgency,
avatar_url AS avatar
SELECT lead_name AS name, assigned_broker,
COALESCE(last_contact_hours_ago, 0)::int AS last_contact_hours_ago,
COALESCE(qd_score, 0)::float AS qd_score, urgency, avatar_url AS avatar
FROM lead_activity_log
WHERE tenant_id = $1
ORDER BY last_contact_hours_ago DESC, qd_score DESC
LIMIT $2
"""
return sql, [ctx.tenant_id, row_limit]
sql = """
SELECT
activity_type AS type,
COALESCE(activity_title, activity_summary, activity_type) AS title,
activity_summary AS summary,
actor_name AS actor,
TO_CHAR(activity_at, 'YYYY-MM-DD HH24:MI') AS date
SELECT activity_type AS type, COALESCE(activity_title, activity_summary, activity_type) AS title,
activity_summary AS summary, actor_name AS actor,
TO_CHAR(activity_at, 'YYYY-MM-DD HH24:MI') AS date
FROM lead_activity_log
WHERE tenant_id = $1
ORDER BY activity_at DESC
@@ -244,27 +231,19 @@ class DataAccessGateway:
if dataset == "crm_contacts_overview":
sql = """
SELECT
p.person_id::text AS id,
p.full_name AS name,
COALESCE(p.primary_email, '') AS email,
COALESCE(p.primary_phone, '') AS phone,
COALESCE(p.city, '') AS city,
COALESCE(p.buyer_type, 'unclassified') AS buyer_type,
COALESCE(q.current_value, 0)::float AS qd_score
SELECT p.person_id::text AS id, p.full_name AS name,
COALESCE(p.primary_email, '') AS email,
COALESCE(p.primary_phone, '') AS phone,
COALESCE(p.city, '') AS city,
COALESCE(p.buyer_type, 'unclassified') AS buyer_type,
COALESCE(q.current_value, 0)::float AS qd_score
FROM crm_people p
LEFT JOIN LATERAL (
SELECT current_value
FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY
CASE
WHEN q.score_type = 'engagement_score' THEN 0
WHEN q.score_type = 'intent_score' THEN 1
WHEN q.score_type = 'urgency_score' THEN 2
ELSE 3
END,
q.computed_at DESC
ORDER BY CASE WHEN q.score_type = 'engagement_score' THEN 0 WHEN q.score_type = 'intent_score' THEN 1 WHEN q.score_type = 'urgency_score' THEN 2 ELSE 3 END,
q.computed_at DESC
LIMIT 1
) q ON TRUE
ORDER BY qd_score DESC, p.full_name ASC
@@ -274,23 +253,10 @@ class DataAccessGateway:
if dataset == "crm_opportunity_pipeline":
sql = """
SELECT
o.stage::text AS stage,
COUNT(*)::int AS count,
COALESCE(SUM(o.value), 0)::float AS value,
COALESCE(
json_agg(
json_build_object(
'id', o.opportunity_id,
'name', p.full_name,
'company', COALESCE(a.account_name, ''),
'value', COALESCE(o.value, 0),
'nextAction', COALESCE(o.next_action, '')
)
ORDER BY o.value DESC NULLS LAST
) FILTER (WHERE o.opportunity_id IS NOT NULL),
'[]'::json
) AS leads
SELECT o.stage::text AS stage, COUNT(*)::int AS count, COALESCE(SUM(o.value), 0)::float AS value,
COALESCE(json_agg(json_build_object('id', o.opportunity_id, 'name', p.full_name, 'company', COALESCE(a.account_name, ''),
'value', COALESCE(o.value, 0), 'nextAction', COALESCE(o.next_action, ''))
ORDER BY o.value DESC NULLS LAST) FILTER (WHERE o.opportunity_id IS NOT NULL), '[]'::json) AS leads
FROM crm_opportunities o
JOIN crm_leads l ON l.lead_id = o.lead_id
JOIN crm_people p ON p.person_id = l.person_id
@@ -301,95 +267,213 @@ class DataAccessGateway:
"""
return sql, [row_limit]
if dataset == "crm_property_interest_rollup":
if dataset == "oracle_property_interest_rollup":
sql = """
SELECT
project_name AS category,
COUNT(*)::int AS value,
ROUND(AVG(COALESCE((budget_min + budget_max) / 2.0, budget_max, budget_min, 0)), 2)::float AS average_budget
FROM crm_property_interests
GROUP BY project_name
ORDER BY value DESC, project_name ASC
SELECT COALESCE(pi.project_name, ip.project_name, 'Unknown Project') AS category,
COUNT(*)::int AS value,
ROUND(AVG(COALESCE((pi.budget_min + pi.budget_max) / 2.0, pi.budget_max, pi.budget_min, 0)), 2)::float AS average_budget,
MAX(pi.created_at) AS latest_interest_at
FROM crm_property_interests pi
LEFT JOIN inventory_projects ip ON ip.project_id = pi.project_id
GROUP BY COALESCE(pi.project_name, ip.project_name, 'Unknown Project')
ORDER BY value DESC, category ASC
LIMIT $1
"""
return sql, [row_limit]
if dataset == "crm_last_interacted_clients":
if dataset == "oracle_last_contacted_clients":
sql = """
SELECT
p.person_id::text AS id,
p.full_name AS name,
COALESCE(p.primary_email, '') AS email,
COALESCE(p.primary_phone, '') AS phone,
COALESCE(MAX(i.happened_at), p.updated_at, p.created_at) AS last_interaction_at,
COUNT(i.interaction_id)::int AS interaction_count,
COALESCE(q.current_value, 0)::float AS qd_score
WITH message_contacts AS (
SELECT i.person_id, MAX(m.delivered_at) AS contacted_at
FROM intel_messages m JOIN intel_interactions i ON i.interaction_id = m.interaction_id
GROUP BY i.person_id
), email_contacts AS (
SELECT i.person_id, MAX(e.sent_at) AS contacted_at
FROM intel_emails e JOIN intel_interactions i ON i.interaction_id = e.interaction_id
GROUP BY i.person_id
), call_contacts AS (
SELECT i.person_id, MAX(i.happened_at) AS contacted_at
FROM intel_calls c JOIN intel_interactions i ON i.interaction_id = c.interaction_id
GROUP BY i.person_id
), visit_contacts AS (
SELECT person_id, MAX(visited_at) AS contacted_at FROM intel_visits GROUP BY person_id
), thread_contacts AS (
SELECT person_id, MAX(last_message_at) AS contacted_at FROM intel_whatsapp_threads GROUP BY person_id
), interaction_contacts AS (
SELECT person_id, MAX(happened_at) AS contacted_at FROM intel_interactions GROUP BY person_id
), next_reminders AS (
SELECT DISTINCT ON (person_id) person_id, title AS next_action, due_at AS next_action_at
FROM intel_reminders
WHERE status IN ('pending', 'open', 'scheduled')
ORDER BY person_id, due_at ASC NULLS LAST
), contact_rollup AS (
SELECT p.person_id,
GREATEST(
COALESCE(mc.contacted_at, '-infinity'::timestamptz),
COALESCE(ec.contacted_at, '-infinity'::timestamptz),
COALESCE(cc.contacted_at, '-infinity'::timestamptz),
COALESCE(vc.contacted_at, '-infinity'::timestamptz),
COALESCE(tc.contacted_at, '-infinity'::timestamptz),
COALESCE(ic.contacted_at, '-infinity'::timestamptz)
) AS last_contacted_at,
mc.contacted_at AS last_message_at, ec.contacted_at AS last_email_at,
cc.contacted_at AS last_call_at, vc.contacted_at AS last_visit_at,
tc.contacted_at AS last_whatsapp_at, ic.contacted_at AS last_interaction_at
FROM crm_people p
LEFT JOIN message_contacts mc ON mc.person_id = p.person_id
LEFT JOIN email_contacts ec ON ec.person_id = p.person_id
LEFT JOIN call_contacts cc ON cc.person_id = p.person_id
LEFT JOIN visit_contacts vc ON vc.person_id = p.person_id
LEFT JOIN thread_contacts tc ON tc.person_id = p.person_id
LEFT JOIN interaction_contacts ic ON ic.person_id = p.person_id
)
SELECT p.person_id::text AS id, p.full_name AS name,
COALESCE(p.primary_email, '') AS email, COALESCE(p.primary_phone, '') AS phone,
NULLIF(cr.last_contacted_at, '-infinity'::timestamptz) AS last_contacted_at,
CASE
WHEN cr.last_contacted_at = cr.last_call_at THEN 'phone'
WHEN cr.last_contacted_at = cr.last_email_at THEN 'email'
WHEN cr.last_contacted_at = cr.last_visit_at THEN 'site_visit'
WHEN cr.last_contacted_at = cr.last_whatsapp_at THEN 'whatsapp'
WHEN cr.last_contacted_at = cr.last_message_at THEN 'message'
WHEN cr.last_contacted_at = cr.last_interaction_at THEN 'interaction'
ELSE 'unknown'
END AS last_contact_channel,
COALESCE(li.summary, nr.next_action, '') AS last_contact_summary,
COUNT(DISTINCT i.interaction_id)::int AS interaction_count,
COALESCE(q.current_value, 0)::float AS qd_score,
COALESCE(nr.next_action, '') AS next_action,
nr.next_action_at
FROM crm_people p
JOIN contact_rollup cr ON cr.person_id = p.person_id
LEFT JOIN intel_interactions i ON i.person_id = p.person_id
LEFT JOIN LATERAL (
SELECT current_value
FROM intel_qd_scores q
SELECT summary
FROM intel_interactions li
WHERE li.person_id = p.person_id
ORDER BY li.happened_at DESC
LIMIT 1
) li ON TRUE
LEFT JOIN next_reminders nr ON nr.person_id = p.person_id
LEFT JOIN LATERAL (
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY
CASE
WHEN q.score_type = 'engagement_score' THEN 0
WHEN q.score_type = 'intent_score' THEN 1
WHEN q.score_type = 'urgency_score' THEN 2
ELSE 3
END,
q.computed_at DESC
ORDER BY q.computed_at DESC
LIMIT 1
) q ON TRUE
GROUP BY p.person_id, p.full_name, p.primary_email, p.primary_phone, p.updated_at, p.created_at, q.current_value
ORDER BY last_interaction_at DESC NULLS LAST, interaction_count DESC, p.full_name ASC
WHERE cr.last_contacted_at <> '-infinity'::timestamptz
GROUP BY p.person_id, p.full_name, p.primary_email, p.primary_phone, cr.last_contacted_at,
cr.last_message_at, cr.last_email_at, cr.last_call_at, cr.last_visit_at,
cr.last_whatsapp_at, cr.last_interaction_at, li.summary, nr.next_action,
nr.next_action_at, q.current_value
ORDER BY last_contacted_at DESC NULLS LAST, interaction_count DESC, p.full_name ASC
LIMIT $1
"""
return sql, [row_limit]
if dataset == "crm_top_interested_clients":
if dataset == "oracle_top_interested_clients":
sql = """
SELECT
p.person_id::text AS id,
p.full_name AS name,
COALESCE(p.primary_email, '') AS email,
COALESCE(p.primary_phone, '') AS phone,
COUNT(pi.interest_id)::int AS interest_count,
STRING_AGG(DISTINCT pi.project_name, ', ' ORDER BY pi.project_name) AS projects,
COALESCE(MAX(pi.created_at), p.updated_at, p.created_at) AS last_interest_at,
COALESCE(q.current_value, 0)::float AS qd_score
WITH interest_mentions AS (
SELECT i.person_id, COUNT(*)::int AS mention_count, MAX(COALESCE(m.delivered_at, i.happened_at)) AS last_mention_at
FROM intel_interactions i
LEFT JOIN intel_messages m ON m.interaction_id = i.interaction_id
WHERE LOWER(COALESCE(i.summary, '') || ' ' || COALESCE(m.message_text, '')) ~
'(interested|interest|shortlist|visit|book|budget|configuration|bhk|project|property)'
GROUP BY i.person_id
)
SELECT p.person_id::text AS id, p.full_name AS name,
COALESCE(p.primary_email, '') AS email, COALESCE(p.primary_phone, '') AS phone,
COUNT(DISTINCT pi.interest_id)::int AS explicit_interest_count,
COALESCE(MAX(im.mention_count), 0)::int AS inferred_interest_count,
(COUNT(DISTINCT pi.interest_id) + COALESCE(MAX(im.mention_count), 0))::int AS interest_count,
STRING_AGG(DISTINCT COALESCE(pi.project_name, ip.project_name), ', ' ORDER BY COALESCE(pi.project_name, ip.project_name)) AS projects,
GREATEST(COALESCE(MAX(pi.created_at), '-infinity'::timestamptz),
COALESCE(MAX(im.last_mention_at), '-infinity'::timestamptz),
COALESCE(p.updated_at, p.created_at)) AS last_interest_at,
COALESCE(q.current_value, 0)::float AS qd_score,
COALESCE(MAX(pi.notes), '') AS latest_interest_note
FROM crm_people p
INNER JOIN crm_property_interests pi ON pi.person_id = p.person_id
LEFT JOIN crm_property_interests pi ON pi.person_id = p.person_id
LEFT JOIN inventory_projects ip ON ip.project_id = pi.project_id
LEFT JOIN interest_mentions im ON im.person_id = p.person_id
LEFT JOIN LATERAL (
SELECT current_value
FROM intel_qd_scores q
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY
CASE
WHEN q.score_type = 'engagement_score' THEN 0
WHEN q.score_type = 'intent_score' THEN 1
WHEN q.score_type = 'urgency_score' THEN 2
ELSE 3
END,
q.computed_at DESC
ORDER BY q.computed_at DESC
LIMIT 1
) q ON TRUE
GROUP BY p.person_id, p.full_name, p.primary_email, p.primary_phone, p.updated_at, p.created_at, q.current_value
HAVING COUNT(DISTINCT pi.interest_id) > 0 OR COALESCE(MAX(im.mention_count), 0) > 0
ORDER BY interest_count DESC, qd_score DESC, last_interest_at DESC NULLS LAST, p.full_name ASC
LIMIT $1
"""
return sql, [row_limit]
if dataset == "crm_interaction_timeline":
if dataset == "oracle_client_interaction_timeline":
sql = """
SELECT
i.interaction_type AS type,
COALESCE(i.summary, i.interaction_type) AS title,
CONCAT(p.full_name, ' · ', i.channel::text) AS summary,
p.full_name AS actor,
TO_CHAR(i.happened_at, 'YYYY-MM-DD HH24:MI') AS date
FROM intel_interactions i
JOIN crm_people p ON p.person_id = i.person_id
ORDER BY i.happened_at DESC
WITH timeline AS (
SELECT i.person_id, i.channel::text AS type, COALESCE(i.interaction_type, i.channel::text) AS title,
COALESCE(i.summary, '') AS detail, i.happened_at AS event_at, 'interaction' AS source_type
FROM intel_interactions i
UNION ALL
SELECT i.person_id, 'message', COALESCE(m.sender_role, 'message'), m.message_text, m.delivered_at, 'message'
FROM intel_messages m JOIN intel_interactions i ON i.interaction_id = m.interaction_id
UNION ALL
SELECT i.person_id, 'call', c.call_direction::text, COALESCE(t.full_text, c.call_outcome, 'Call record'), i.happened_at, 'call'
FROM intel_calls c
JOIN intel_interactions i ON i.interaction_id = c.interaction_id
LEFT JOIN intel_transcripts t ON t.call_id = c.call_id OR t.interaction_id = i.interaction_id
UNION ALL
SELECT i.person_id, 'email', COALESCE(e.subject, 'Email'), COALESCE(e.body_text, ''), e.sent_at, 'email'
FROM intel_emails e JOIN intel_interactions i ON i.interaction_id = e.interaction_id
UNION ALL
SELECT v.person_id, 'site_visit', COALESCE(v.project_name, 'Site visit'), COALESCE(v.visit_notes, ''), v.visited_at, 'visit'
FROM intel_visits v
UNION ALL
SELECT r.person_id, 'reminder', r.title, COALESCE(r.notes, r.status), COALESCE(r.due_at, r.created_at), 'reminder'
FROM intel_reminders r
UNION ALL
SELECT q.person_id, 'qd_score', q.score_type, COALESCE(q.reasoning, q.current_value::text), q.computed_at, 'qd_score'
FROM intel_qd_scores q
UNION ALL
SELECT qt.person_id, 'qd_timeseries', COALESCE(qt.signal_source, qt.score_type), qt.value::text, qt.timestamp, 'qd_timeseries'
FROM intel_qd_timeseries qt
)
SELECT t.type, t.title, CONCAT(p.full_name, ' - ', t.detail) AS summary,
p.full_name AS actor, TO_CHAR(t.event_at, 'YYYY-MM-DD HH24:MI') AS date,
t.source_type, t.event_at
FROM timeline t
JOIN crm_people p ON p.person_id = t.person_id
ORDER BY t.event_at DESC NULLS LAST
LIMIT $1
"""
return sql, [row_limit]
if dataset == "oracle_client_360_summary":
sql = """
SELECT p.person_id::text AS id, p.full_name AS name,
COALESCE(p.primary_email, '') AS email, COALESCE(p.primary_phone, '') AS phone,
COALESCE(l.status::text, 'unknown') AS lead_status,
COALESCE(l.budget_band, '') AS budget_band,
COALESCE(l.urgency, '') AS urgency,
COALESCE(q.current_value, 0)::float AS qd_score,
COUNT(DISTINCT pi.interest_id)::int AS interest_count,
COUNT(DISTINCT i.interaction_id)::int AS interaction_count,
MAX(i.happened_at) AS last_interaction_at,
STRING_AGG(DISTINCT COALESCE(pi.project_name, ip.project_name), ', ' ORDER BY COALESCE(pi.project_name, ip.project_name)) AS projects
FROM crm_people p
LEFT JOIN crm_leads l ON l.person_id = p.person_id
LEFT JOIN crm_property_interests pi ON pi.person_id = p.person_id
LEFT JOIN inventory_projects ip ON ip.project_id = pi.project_id
LEFT JOIN intel_interactions i ON i.person_id = p.person_id
LEFT JOIN LATERAL (
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY q.computed_at DESC
LIMIT 1
) q ON TRUE
GROUP BY p.person_id, p.full_name, p.primary_email, p.primary_phone, l.status, l.budget_band, l.urgency, q.current_value
ORDER BY qd_score DESC, interaction_count DESC, interest_count DESC, name ASC
LIMIT $1
"""
return sql, [row_limit]

468
backend/oracle/natural_db_agent.py Normal file
View File

@@ -0,0 +1,468 @@
"""
Natural DB-first Oracle agent.
The LLM can plan arbitrary analytical SELECT statements over the Velocity CRM,
intel, inventory, and read-model tables. The executor enforces a read-only SQL
contract and a UI row cap; write paths stay behind typed API endpoints.
"""
from __future__ import annotations
import json
import logging
import os
import re
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from backend.services.runtime_llm_service import runtime_llm_service
try:
import asyncpg # type: ignore
except Exception: # pragma: no cover
asyncpg = None # type: ignore
logger = logging.getLogger(__name__)
MAX_ROW_CAP = 500
ALLOWED_TABLES = {
"crm_people", "crm_leads", "crm_accounts", "crm_households", "crm_relationships",
"crm_opportunities", "crm_property_interests", "crm_stage_history",
"intel_interactions", "intel_messages", "intel_calls", "intel_transcripts",
"intel_emails", "intel_email_threads", "intel_whatsapp_threads", "intel_visits",
"intel_reminders", "intel_qd_scores", "intel_qd_timeseries",
"intel_extracted_facts", "intel_call_objections", "intel_cctv_links",
"intel_perception_events", "intel_vehicle_events",
"inventory_projects", "inventory_units",
"read_last_contacted", "read_next_best_action",
}
DESTRUCTIVE_SQL = re.compile(
r"\b(insert|update|delete|drop|alter|truncate|copy|create|grant|revoke|call|execute|do|merge)\b",
re.IGNORECASE,
)
TABLE_REF_RE = re.compile(r"\b(?:from|join)\s+([a-zA-Z_][\w.]*)(?:\s|$)", re.IGNORECASE)
def _json_safe(value: Any) -> Any:
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Decimal):
return float(value)
if isinstance(value, (list, tuple)):
return [_json_safe(v) for v in value]
if isinstance(value, dict):
return {str(k): _json_safe(v) for k, v in value.items()}
return value
def db_ready() -> bool:
if asyncpg is None:
return False
database_url = os.getenv("DATABASE_URL", "")
return bool(database_url and not database_url.startswith("PLACEHOLDER")) or all(
os.getenv(name) for name in ("VELOCITY_DB_NAME", "VELOCITY_DB_USER", "VELOCITY_DB_PASSWORD")
)
async def connect_db() -> Any:
if asyncpg is None:
raise RuntimeError("asyncpg is not installed.")
database_url = os.getenv("DATABASE_URL", "")
if database_url and not database_url.startswith("PLACEHOLDER"):
return await asyncpg.connect(database_url)
return await asyncpg.connect(
host=os.getenv("VELOCITY_DB_HOST", "127.0.0.1"),
port=int(os.getenv("VELOCITY_DB_PORT", "5432")),
database=os.environ["VELOCITY_DB_NAME"],
user=os.environ["VELOCITY_DB_USER"],
password=os.environ["VELOCITY_DB_PASSWORD"],
)
@dataclass
class NaturalQueryResult:
prompt: str
sql: str
title: str
summary: str
columns: list[str]
rows: list[dict[str, Any]]
row_count: int
source_tables: list[str]
component_type: str
warnings: list[str]
def as_dict(self) -> dict[str, Any]:
return {
"prompt": self.prompt,
"sql": self.sql,
"title": self.title,
"summary": self.summary,
"columns": self.columns,
"rows": self.rows,
"rowCount": self.row_count,
"sourceTables": self.source_tables,
"componentType": self.component_type,
"warnings": self.warnings,
}
def sanitize_sql(sql: str, row_limit: int) -> tuple[str, list[str], list[str]]:
warnings: list[str] = []
clean = re.sub(r"--.*?$|/\*.*?\*/", "", sql.strip(), flags=re.MULTILINE | re.DOTALL).strip().rstrip(";")
if not re.match(r"^(select|with)\b", clean, re.IGNORECASE):
raise ValueError("Oracle SQL agent only accepts SELECT or WITH queries.")
if DESTRUCTIVE_SQL.search(clean):
raise ValueError("Oracle SQL agent blocked non-read SQL.")
tables = []
for match in TABLE_REF_RE.finditer(clean):
table = match.group(1).split(".")[-1].strip('"').lower()
if table in {"lateral", "select"}:
continue
if table and table not in tables:
tables.append(table)
blocked = [table for table in tables if table not in ALLOWED_TABLES]
if blocked:
raise ValueError(f"Oracle SQL agent blocked unknown tables: {', '.join(blocked)}")
capped = max(1, min(int(row_limit or 100), MAX_ROW_CAP))
if not re.search(r"\blimit\s+\d+\b", clean, re.IGNORECASE):
clean = f"SELECT * FROM ({clean}) oracle_limited_rows LIMIT {capped}"
warnings.append(f"Applied UI row cap LIMIT {capped}.")
return clean, tables, warnings
def infer_component_type(prompt: str, columns: list[str], rows: list[dict[str, Any]]) -> str:
lower = prompt.lower()
if any(term in lower for term in ("timeline", "conversation", "whatsapp", "message", "call", "email", "history")):
return "activity_stream"
if len(rows) == 1 and len(columns) <= 5 and any(isinstance(rows[0].get(c), (int, float)) for c in columns):
return "kpi_tile"
if any(c.endswith("_at") or c in {"date", "when", "timestamp", "happened_at"} for c in columns):
if len(rows) > 1 and any(term in lower for term in ("trend", "over time", "timeseries")):
return "line_chart"
if any(term in lower for term in ("timeline", "activity", "last", "recent")):
return "activity_stream"
numeric_cols = [c for c in columns if rows and isinstance(rows[0].get(c), (int, float))]
if numeric_cols and any(term in lower for term in ("count", "compare", "distribution", "most", "top", "by ")):
return "bar_chart"
return "table"
def title_from_prompt(prompt: str) -> str:
words = re.sub(r"\s+", " ", prompt.strip()).strip(" ?.!")
return words[:1].upper() + words[1:80] if words else "Oracle Query Result"
class NaturalDbAgent:
async def schema_catalog(self, conn: Any | None = None) -> dict[str, Any]:
own_conn = conn is None
if conn is None:
if not db_ready():
return {"tables": [], "available": False}
conn = await connect_db()
try:
rows = await conn.fetch(
"""
SELECT c.table_name, c.column_name, c.data_type, c.udt_name, c.is_nullable
FROM information_schema.columns c
WHERE c.table_schema = 'public' AND c.table_name = ANY($1::text[])
ORDER BY c.table_name, c.ordinal_position
""",
sorted(ALLOWED_TABLES),
)
counts = {}
for table in sorted(ALLOWED_TABLES):
exists = await conn.fetchval("SELECT to_regclass($1)", f"public.{table}")
counts[table] = None if not exists else int(await conn.fetchval(f"SELECT COUNT(*) FROM {table}"))
tables: dict[str, dict[str, Any]] = {}
for row in rows:
entry = tables.setdefault(row["table_name"], {"columns": [], "rowCount": counts.get(row["table_name"])})
entry["columns"].append({
"name": row["column_name"],
"dataType": row["data_type"],
"udtName": row["udt_name"],
"nullable": row["is_nullable"] == "YES",
})
return {"available": True, "tables": tables, "allowedTables": sorted(ALLOWED_TABLES)}
finally:
if own_conn:
await conn.close()
async def data_health(self, conn: Any | None = None) -> dict[str, Any]:
catalog = await self.schema_catalog(conn)
expected = {
"crm_people": 341,
"crm_leads": 250,
"crm_opportunities": 400,
"crm_property_interests": 400,
"intel_interactions": 1897,
"intel_messages": 6944,
"intel_calls": 478,
"intel_transcripts": 231,
"intel_emails": 149,
"intel_visits": 305,
"intel_reminders": 759,
"intel_extracted_facts": 1686,
"read_last_contacted": 250,
"read_next_best_action": 250,
}
tables = catalog.get("tables", {})
counts = {table: (tables.get(table) or {}).get("rowCount") for table in sorted(ALLOWED_TABLES)}
return {
"counts": counts,
"expectedSyntheticV2Counts": expected,
"missingTables": [t for t, count in counts.items() if count is None],
"emptyTables": [t for t, count in counts.items() if count == 0],
"belowExpected": {t: {"expected": e, "actual": counts.get(t)} for t, e in expected.items() if (counts.get(t) or 0) < e},
}
async def execute_prompt(self, prompt: str, *, row_limit: int = 100, conn: Any | None = None) -> NaturalQueryResult:
if not prompt.strip():
raise ValueError("Prompt is required.")
own_conn = conn is None
if conn is None:
if not db_ready():
raise RuntimeError("Database unavailable for Oracle natural query.")
conn = await connect_db()
try:
catalog = await self.schema_catalog(conn)
plan = await self._plan_sql(prompt, catalog, row_limit)
return await self._run_plan(conn, prompt, plan, row_limit)
finally:
if own_conn:
await conn.close()
async def _run_plan(self, conn: Any, prompt: str, plan: dict[str, Any], row_limit: int) -> NaturalQueryResult:
raw_sql = str(plan.get("sql") or "").strip()
if not raw_sql:
raw_sql = self._fallback_sql(prompt, row_limit)
sql, tables, warnings = sanitize_sql(raw_sql, row_limit)
try:
records = await conn.fetch(sql)
except Exception as exc:
retry = await self._repair_sql(prompt, raw_sql, str(exc), row_limit)
sql, tables, retry_warnings = sanitize_sql(retry, row_limit)
warnings.extend(retry_warnings)
warnings.append(f"Initial SQL repaired after database error: {exc}")
records = await conn.fetch(sql)
if not records:
retry_sql = self._zero_row_retry_sql(prompt, row_limit, raw_sql)
if retry_sql and retry_sql.strip() != raw_sql.strip():
retry_clean, retry_tables, retry_warnings = sanitize_sql(retry_sql, row_limit)
retry_records = await conn.fetch(retry_clean)
if retry_records:
sql = retry_clean
tables = retry_tables
records = retry_records
warnings.extend(retry_warnings)
warnings.append("Initial SQL returned zero rows; Oracle retried with a broader CRM read query.")
rows = [_json_safe(dict(record)) for record in records]
columns = list(rows[0].keys()) if rows else []
component_type = infer_component_type(prompt, columns, rows)
return NaturalQueryResult(
prompt=prompt,
sql=sql,
title=str(plan.get("title") or title_from_prompt(prompt)),
summary=str(plan.get("rationale") or f"SQL-backed Oracle result from {', '.join(tables) or 'Velocity CRM'}."),
columns=columns,
rows=rows,
row_count=len(rows),
source_tables=tables,
component_type=component_type,
warnings=warnings,
)
async def _plan_sql(self, prompt: str, catalog: dict[str, Any], row_limit: int) -> dict[str, Any]:
fallback = {"sql": self._fallback_sql(prompt, row_limit), "title": title_from_prompt(prompt), "rationale": "Deterministic SQL planner fallback."}
try:
providers = runtime_llm_service._provider_catalog()
except Exception:
providers = {}
if not providers:
return fallback
schema_brief = json.dumps(catalog.get("tables", {}), default=str)[:16000]
system = (
"You are Oracle's read-only PostgreSQL planner. Generate one useful SELECT or WITH query "
"for the user's CRM question. Use only the provided schema. Return JSON with sql, title, rationale. "
"Never generate INSERT, UPDATE, DELETE, DDL, COPY, or permission statements."
)
try:
response = await runtime_llm_service.chat(
provider_id="sglang",
model=None,
system_prompt=system,
messages=[{"role": "user", "content": f"Schema:\n{schema_brief}\n\nQuestion:\n{prompt}\n\nRow cap: {row_limit}"}],
temperature=0.05,
response_format="json",
metadata={"agent": "oracle_natural_db_agent"},
)
message = response.get("message") or {}
parsed = message.get("parsedJson")
content = message.get("content") or "{}"
if not isinstance(parsed, dict):
parsed = json.loads(content) if isinstance(content, str) else content
if isinstance(parsed, dict) and parsed.get("sql"):
return parsed
except Exception as exc:
logger.warning("Natural DB planner LLM failed, using fallback: %s", exc)
return fallback
async def _repair_sql(self, prompt: str, failed_sql: str, error: str, row_limit: int) -> str:
# Keep retry operationally deterministic if model is unavailable.
if "read_last_contacted" in failed_sql and "does not exist" in error.lower():
return self._base_last_contacted_sql(row_limit)
if "read_next_best_action" in failed_sql and "does not exist" in error.lower():
return self._base_last_contacted_sql(row_limit)
return self._fallback_sql(prompt, row_limit)
def _zero_row_retry_sql(self, prompt: str, row_limit: int, previous_sql: str) -> str | None:
lower = prompt.lower()
if any(term in lower for term in ("contact", "recent", "last", "call", "message", "email", "whatsapp", "follow")):
return self._base_last_contacted_sql(row_limit)
if any(term in lower for term in ("interest", "interested", "property", "project", "unit", "budget", "bhk")):
return self._base_property_interest_sql(row_limit)
if "from crm_people" not in previous_sql.lower():
return self._generic_clients_sql(row_limit)
return None
def _base_last_contacted_sql(self, row_limit: int) -> str:
limit = max(1, min(row_limit, MAX_ROW_CAP))
return f"""
WITH contact_events AS (
SELECT i.person_id, i.happened_at AS event_at, i.channel::text AS channel,
i.interaction_type AS event_type, i.summary AS summary, i.broker_name AS actor
FROM intel_interactions i
WHERE i.happened_at IS NOT NULL
UNION ALL
SELECT i.person_id, m.delivered_at, 'message', COALESCE(m.sender_role, 'message'), m.message_text, m.sender_name
FROM intel_messages m
JOIN intel_interactions i ON i.interaction_id = m.interaction_id
WHERE m.delivered_at IS NOT NULL
UNION ALL
SELECT i.person_id, e.sent_at, 'email', COALESCE(e.direction::text, 'email'), e.subject, e.from_address
FROM intel_emails e
JOIN intel_interactions i ON i.interaction_id = e.interaction_id
WHERE e.sent_at IS NOT NULL
UNION ALL
SELECT v.person_id, v.visited_at, 'site_visit', 'visit', v.outcome, v.hosted_by
FROM intel_visits v
WHERE v.visited_at IS NOT NULL
),
ranked AS (
SELECT *, row_number() OVER (PARTITION BY person_id ORDER BY event_at DESC) AS rn,
count(*) OVER (PARTITION BY person_id) AS interaction_count
FROM contact_events
)
SELECT p.person_id::text, p.full_name AS name, p.primary_phone AS phone,
p.primary_email AS email, r.event_at AS last_contacted_at,
r.channel AS last_contact_channel, r.event_type AS last_interaction_type,
r.summary AS last_contact_summary, r.actor AS last_contact_actor,
r.interaction_count::int,
q.current_value AS qd_score
FROM ranked r
JOIN crm_people p ON p.person_id = r.person_id
LEFT JOIN LATERAL (
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY q.current_value DESC, q.computed_at DESC
LIMIT 1
) q ON TRUE
WHERE r.rn = 1
ORDER BY r.event_at DESC
LIMIT {limit}
"""
def _base_property_interest_sql(self, row_limit: int) -> str:
limit = max(1, min(row_limit, MAX_ROW_CAP))
return f"""
SELECT p.person_id::text, p.full_name AS name, p.primary_phone AS phone, p.primary_email AS email,
COUNT(pi.interest_id)::int AS interest_count,
string_agg(DISTINCT COALESCE(pi.project_name, pr.project_name), ', ') AS projects,
string_agg(DISTINCT pi.configuration, ', ') AS configurations,
MIN(pi.budget_min) AS budget_min, MAX(pi.budget_max) AS budget_max,
MAX(pi.last_discussed_at) AS last_interest_at,
MAX(q.current_value) AS qd_score
FROM crm_people p
JOIN crm_property_interests pi ON pi.person_id = p.person_id
LEFT JOIN inventory_projects pr ON pr.project_id = pi.project_id
LEFT JOIN intel_qd_scores q ON q.person_id = p.person_id
GROUP BY p.person_id, p.full_name, p.primary_phone, p.primary_email
HAVING COUNT(pi.interest_id) > 0
ORDER BY interest_count DESC, qd_score DESC NULLS LAST, last_interest_at DESC NULLS LAST
LIMIT {limit}
"""
def _generic_clients_sql(self, row_limit: int) -> str:
limit = max(1, min(row_limit, MAX_ROW_CAP))
return f"""
SELECT p.person_id::text, p.full_name AS name, p.primary_email AS email, p.primary_phone AS phone,
p.buyer_type, l.status::text AS lead_status, l.budget_band, l.urgency,
q.current_value AS qd_score
FROM crm_people p
LEFT JOIN LATERAL (
SELECT * FROM crm_leads l WHERE l.person_id = p.person_id ORDER BY l.updated_at DESC LIMIT 1
) l ON TRUE
LEFT JOIN LATERAL (
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY q.current_value DESC, q.computed_at DESC
LIMIT 1
) q ON TRUE
ORDER BY qd_score DESC NULLS LAST, p.full_name ASC
LIMIT {limit}
"""
def _fallback_sql(self, prompt: str, row_limit: int) -> str:
lower = prompt.lower()
limit = max(1, min(row_limit, MAX_ROW_CAP))
if "objection" in lower:
return f"""
SELECT p.person_id::text, p.full_name AS name, co.objection_type, co.category, co.severity,
co.client_quote, co.agent_response, co.extracted_at
FROM intel_call_objections co
JOIN intel_calls c ON c.call_id = co.call_id
JOIN intel_interactions i ON i.interaction_id = c.interaction_id
JOIN crm_people p ON p.person_id = i.person_id
ORDER BY co.extracted_at DESC
LIMIT {limit}
"""
if "whatsapp" in lower or "message" in lower or "conversation" in lower:
return f"""
SELECT p.person_id::text, p.full_name AS name, 'whatsapp' AS type,
m.message_text AS summary, m.sender_role AS actor, m.delivered_at AS date
FROM intel_messages m
JOIN intel_interactions i ON i.interaction_id = m.interaction_id
JOIN crm_people p ON p.person_id = i.person_id
WHERE lower(m.message_text) LIKE '%' || lower(split_part($${prompt}$$, ' ', 1)) || '%'
OR i.channel = 'whatsapp'
ORDER BY m.delivered_at DESC
LIMIT {limit}
"""
if "contact" in lower or "recent" in lower or "last" in lower:
return f"""
SELECT p.person_id::text, p.full_name AS name, p.primary_phone AS phone,
lc.last_contact_at AS last_contacted_at, lc.last_channel AS last_contact_channel,
lc.last_interaction_type, lc.days_since_contact, lc.total_interactions AS interaction_count,
nba.recommended_action AS next_action, q.current_value AS qd_score
FROM crm_people p
LEFT JOIN read_last_contacted lc ON lc.person_id = p.person_id
LEFT JOIN read_next_best_action nba ON nba.person_id = p.person_id
LEFT JOIN LATERAL (
SELECT current_value FROM intel_qd_scores q
WHERE q.person_id = p.person_id
ORDER BY q.current_value DESC, q.computed_at DESC
LIMIT 1
) q ON TRUE
WHERE lc.last_contact_at IS NOT NULL
ORDER BY lc.last_contact_at DESC
LIMIT {limit}
"""
if "4 bhk" in lower or "budget" in lower or "interest" in lower or "property" in lower or "client" in lower:
return self._base_property_interest_sql(limit)
return self._generic_clients_sql(limit)
natural_db_agent = NaturalDbAgent()

252
backend/oracle/prompt_orchestrator.py
View File

@@ -10,6 +10,7 @@ import logging
import os
import uuid
import json
import re
from datetime import datetime, timezone
from typing import Any
@@ -18,6 +19,7 @@ from .canvas_service import canvas_service
from .data_access_gateway import data_access_gateway
from .persona_service import persona_service
from .codebook_service import codebook_service, CodebookExample
from .natural_db_agent import natural_db_agent
from backend.services.runtime_llm_service import runtime_llm_service
from backend.services.nemoclaw_runtime import nemoclaw_runtime
@@ -107,7 +109,7 @@ def _build_demo_retrieval_plan(
Produces a valid retrieval plan that passes policy validation.
"""
component_types = _detect_component_types(prompt)
row_limit = 50 if actor_role in ("senior_broker", "junior_broker") else 200
row_limit = _parse_prompt_row_limit(prompt, actor_role)
return {
"planId": str(uuid.uuid4()),
@@ -130,12 +132,12 @@ def _build_demo_retrieval_plan(
_DATASET_MAP: dict[str, str] = {
"pipeline_board": "crm_opportunity_pipeline",
"bar_chart": "crm_property_interest_rollup",
"bar_chart": "oracle_property_interest_rollup",
"geo_map": "lead_geo_interest_rollup",
"table": "crm_contacts_overview",
"line_chart": "crm_property_interest_rollup",
"line_chart": "oracle_property_interest_rollup",
"kpi_tile": "oracle_aggregated_metric",
"activity_stream": "crm_interaction_timeline",
"activity_stream": "oracle_client_interaction_timeline",
}
_CODEBOOK_COMPONENT_MAP: dict[str, str] = {
@@ -164,34 +166,85 @@ def _component_plan_type_from_codebook(example: CodebookExample) -> str:
return _CODEBOOK_COMPONENT_MAP.get(example.component_type, "table")
def _parse_prompt_row_limit(prompt: str, actor_role: str) -> int:
default_limit = 50 if actor_role in ("senior_broker", "junior_broker") else 200
match = re.search(r"\b(?:top|last|latest|recent|first|show|name of the last)\s+(\d{1,4})\b", prompt.lower())
if not match:
return default_limit
requested = max(1, int(match.group(1)))
return min(requested, default_limit)
def _prompt_data_intent(prompt: str) -> str | None:
lowered = prompt.lower()
contact_terms = (
"last contacted", "last contact", "last contacted us", "recently contacted",
"recent contacts", "last call", "last called", "last message", "last messaged",
"last whatsapp", "who contacted us", "contacted us", "contacted clients",
"client contacted", "clients contacted", "follow-up", "follow up",
)
interest_terms = (
"shown interest", "showed interest", "interested clients", "interested client",
"property interest", "project interest", "interested in any", "interest in any",
"interested in our properties", "interested in properties",
)
timeline_terms = (
"conversation", "timeline", "whatsapp", "messages", "message history",
"call history", "transcript", "email", "visit history", "interaction history",
)
client_360_terms = ("client 360", "client dossier", "highest intent buyer", "client profile")
if any(term in lowered for term in contact_terms) or re.search(r"\blast\s+\d+\s+contacted\b", lowered):
return "last_contacted"
if any(term in lowered for term in interest_terms) or (
any(term in lowered for term in ("interest", "interested", "project", "property", "properties"))
and any(term in lowered for term in ("client", "clients", "contact", "contacts"))
):
return "interested_clients"
if any(term in lowered for term in client_360_terms):
return "client_360"
if any(term in lowered for term in timeline_terms):
return "timeline"
return None
def _dataset_for_codebook(example: CodebookExample, prompt: str, component_plan_type: str | None = None) -> str:
chapter = example.chapter_name.lower()
subchapter = example.subchapter_name.lower()
component_plan_type = component_plan_type or _component_plan_type_from_codebook(example)
lowered_prompt = prompt.lower()
data_intent = _prompt_data_intent(prompt)
if data_intent == "last_contacted":
return "oracle_last_contacted_clients" if component_plan_type != "activity_stream" else "oracle_client_interaction_timeline"
if data_intent == "interested_clients":
return "oracle_top_interested_clients" if component_plan_type == "table" else "oracle_property_interest_rollup"
if data_intent == "client_360":
return "oracle_client_360_summary"
if data_intent == "timeline":
return "oracle_client_interaction_timeline"
if component_plan_type == "activity_stream":
return "crm_interaction_timeline"
return "oracle_client_interaction_timeline"
if component_plan_type == "pipeline_board":
return "crm_opportunity_pipeline"
if component_plan_type == "table" and any(term in lowered_prompt for term in ("last interacted", "last interaction", "recently contacted", "recent interaction")):
return "crm_last_interacted_clients"
return "oracle_last_contacted_clients"
if component_plan_type == "table" and any(term in lowered_prompt for term in ("interest", "interested", "project", "property", "properties")) and any(term in lowered_prompt for term in ("client", "clients", "contact", "contacts")):
return "crm_top_interested_clients"
return "oracle_top_interested_clients"
if component_plan_type == "line_chart" and any(term in lowered_prompt for term in ("trend", "time", "history", "growth")):
return "crm_property_interest_rollup"
return "oracle_property_interest_rollup"
if any(term in lowered_prompt for term in ("contact", "client 360", "crm", "account", "lead")):
if "timeline" in lowered_prompt or "message" in lowered_prompt or "call" in lowered_prompt or "email" in lowered_prompt:
return "crm_interaction_timeline"
return "oracle_client_interaction_timeline"
if "pipeline" in lowered_prompt or "opportunit" in lowered_prompt:
return "crm_opportunity_pipeline"
if ("interest" in lowered_prompt or "project" in lowered_prompt or "property" in lowered_prompt) and ("client" in lowered_prompt or "contact" in lowered_prompt):
return "crm_top_interested_clients"
return "oracle_top_interested_clients"
if "interest" in lowered_prompt or "project" in lowered_prompt or "property" in lowered_prompt:
return "crm_property_interest_rollup"
return "oracle_property_interest_rollup"
if "last interacted" in lowered_prompt or "recently contacted" in lowered_prompt or "recent interaction" in lowered_prompt:
return "crm_last_interacted_clients"
return "oracle_last_contacted_clients"
return "crm_contacts_overview"
if "client" in chapter or "client" in subchapter or "contact" in subchapter:
@@ -199,9 +252,9 @@ def _dataset_for_codebook(example: CodebookExample, prompt: str, component_plan_
if "opportun" in chapter or "pipeline" in subchapter:
return "crm_opportunity_pipeline"
if "interaction" in chapter or "communication" in chapter or "timeline" in subchapter:
return "crm_interaction_timeline"
return "oracle_client_interaction_timeline"
if "property" in chapter or "inventory" in chapter or "interest" in subchapter:
return "crm_property_interest_rollup"
return "oracle_property_interest_rollup"
return _DATASET_MAP.get(component_plan_type, "oracle_aggregated_metric")
@@ -211,7 +264,7 @@ def _build_codebook_retrieval_plan(
actor_role: str,
matches: list[CodebookExample],
) -> dict[str, Any]:
row_limit = 50 if actor_role in ("senior_broker", "junior_broker") else 200
row_limit = _parse_prompt_row_limit(prompt, actor_role)
desired_types = _detect_component_types(prompt)
if not desired_types:
desired_types = [_component_plan_type_from_codebook(matches[0])] if matches else ["table"]
@@ -265,12 +318,17 @@ def _title_for_dataset(dataset: str, component_plan_type: str, prompt: str) -> s
"crm_interaction_timeline": "Client Interaction Timeline",
"crm_last_interacted_clients": "Last Interacted Clients",
"crm_top_interested_clients": "Top Interested Clients",
"oracle_property_interest_rollup": "Property Interest Rollup",
"oracle_client_interaction_timeline": "Client Interaction Timeline",
"oracle_last_contacted_clients": "Last Contacted Clients",
"oracle_top_interested_clients": "Top Interested Clients",
"oracle_client_360_summary": "Client 360 Summary",
"broker_performance": "Broker Performance",
}
if dataset == "crm_top_interested_clients" and "top" in lowered_prompt:
if dataset in {"crm_top_interested_clients", "oracle_top_interested_clients"} and "top" in lowered_prompt:
return "Top Interested Clients"
if dataset == "crm_last_interacted_clients" and ("top" in lowered_prompt or "last" in lowered_prompt):
return "Last Interacted Clients"
if dataset in {"crm_last_interacted_clients", "oracle_last_contacted_clients"} and ("top" in lowered_prompt or "last" in lowered_prompt):
return "Last Contacted Clients"
return dataset_titles.get(dataset)
@@ -288,6 +346,11 @@ _RUNTIME_ALLOWED_DATASETS = {
"crm_interaction_timeline",
"crm_last_interacted_clients",
"crm_top_interested_clients",
"oracle_property_interest_rollup",
"oracle_client_interaction_timeline",
"oracle_last_contacted_clients",
"oracle_top_interested_clients",
"oracle_client_360_summary",
}
@@ -348,6 +411,64 @@ class PromptOrchestrator:
await self._persist_execution(execution)
# ── Step 1: Build retrieval plan ──────────────────────────────────────
page = await canvas_service.get_page(page_id, tenant_id)
existing_comps = page.get("components", []) if page else []
next_order_base = self._next_order_base(existing_comps)
section_id = f"sec_prompt_generated_{execution_id.replace('-', '')[:12]}"
natural_result = None
try:
natural_result = await natural_db_agent.execute_prompt(
prompt,
row_limit=_parse_prompt_row_limit(prompt, actor_role),
)
except Exception as exc:
logger.warning("ORCH natural DB agent unavailable, falling back to component planner: %s", exc)
warnings.append(f"Natural DB agent unavailable ({exc}); using component planner fallback.")
if natural_result is not None:
execution["status"] = "executing"
execution["retrievalPlan"] = {
"planId": str(uuid.uuid4()),
"planner": "oracle_natural_db_agent",
"sql": natural_result.sql,
"sourceTables": natural_result.source_tables,
"rowCount": natural_result.row_count,
}
viz_plan = self._build_natural_visualization_plan(
result=natural_result.as_dict(),
prompt=prompt,
execution_id=execution_id,
actor_id=actor_id,
branch_id=branch_id,
base_order=next_order_base,
section_id=section_id,
)
execution["visualizationPlan"] = viz_plan
execution["componentsCreated"] = [c["componentId"] for c in viz_plan.get("components", [])]
try:
if page:
revision = await canvas_service.commit_revision(
page_id=page_id,
tenant_id=tenant_id,
actor_id=actor_id,
commit_kind="prompt",
commit_summary=f"Oracle: {prompt[:80]}",
components=existing_comps + viz_plan.get("components", []),
execution_id=execution_id,
idempotency_key=client_request_id,
)
execution["headRevision"] = revision["revisionNumber"]
except Exception as exc:
logger.warning("ORCH natural revision_commit failed (non-fatal): %s", exc)
warnings.append("Revision commit deferred; will retry on next sync.")
execution["status"] = "completed"
execution["summary"] = self._generate_summary(prompt, viz_plan)
execution["completedAt"] = _now()
execution["warnings"] = warnings + natural_result.warnings
await self._persist_execution(execution)
return execution
codebook_matches = codebook_service.search_examples(prompt, limit=4)
execution["codebookMatches"] = [
{
@@ -580,6 +701,92 @@ class PromptOrchestrator:
return {"components": components}
def _build_natural_visualization_plan(
self,
*,
result: dict[str, Any],
prompt: str,
execution_id: str,
actor_id: str,
branch_id: str,
base_order: int,
section_id: str,
) -> dict[str, Any]:
rows = result.get("rows") or []
columns = result.get("columns") or (list(rows[0].keys()) if rows else [])
ctype = str(result.get("componentType") or "table")
mapped_type = self._map_type(ctype)
dataset = "oracle_natural_sql"
component_id = str(uuid.uuid4())
comp: dict[str, Any] = {
"componentId": component_id,
"type": mapped_type,
"title": result.get("title") or self._generate_title(prompt, ctype),
"description": f"SQL-backed Oracle result from: \"{prompt[:96]}\"",
"dataSourceDescriptor": {
"descriptorId": str(uuid.uuid4()),
"sourceType": "postgres",
"connectorId": "velocity-core-postgres",
"dataset": dataset,
"authContextRef": f"authctx_{actor_id}_scope",
"queryTemplate": result.get("sql", ""),
"queryParameters": {},
"rowLimit": len(rows),
"privacyTier": "standard",
"cachePolicy": {"mode": "revision_scoped"},
},
"visualizationParameters": {
**self._default_viz_params(ctype, dataset, rows),
"columns": columns,
"sqlSummary": result.get("summary"),
"sourceTables": result.get("sourceTables", []),
"rowCount": result.get("rowCount", len(rows)),
},
"dataBindings": self._default_bindings(ctype),
"version": 1,
"lifecycleState": "active",
"provenance": {
"originType": "prompt_generated",
"promptExecutionId": execution_id,
"sourceBranchId": branch_id,
"createdBy": actor_id,
"createdAt": _iso(_now()),
"sourceTables": result.get("sourceTables", []),
"sqlSummary": result.get("summary"),
},
"renderingHints": self._rendering_hints(ctype),
"layout": {
"orderIndex": base_order + 100,
"sectionId": section_id,
"widthMode": "full" if mapped_type in ("table", "pipelineBoard", "timeline", "activityStream") else "half",
"minHeightPx": 320,
"stickyHeader": False,
},
"accessControls": {
"visibilityScope": "private",
"allowedRoles": ["senior_broker", "sales_director", "marketing_operator", "data_steward", "compliance_reviewer", "platform_admin"],
"redactionPolicy": "none",
},
"styleSignature": {
"theme": "velocity_glass",
"paletteToken": "ocean_signal",
"motionProfile": "calm_reveal",
"density": "comfortable",
"radiusScale": "lg",
"typographyScale": "balanced",
},
"validationState": {
"schema": "pass",
"policy": "pass",
"a11y": "pass",
"performance": "pass",
"status": "validated",
},
"auditLog": [f"aud_{execution_id}_natural_sql"],
"dataRows": rows,
}
return {"components": [comp]}
@staticmethod
def _next_order_base(existing_components: list[dict[str, Any]]) -> int:
max_existing = 0
@@ -706,6 +913,9 @@ class PromptOrchestrator:
"crm_contacts_overview": ["name", "email", "phone", "city", "buyer_type", "qd_score"],
"crm_last_interacted_clients": ["name", "email", "phone", "last_interaction_at", "interaction_count", "qd_score"],
"crm_top_interested_clients": ["name", "email", "phone", "interest_count", "projects", "qd_score"],
"oracle_last_contacted_clients": ["name", "phone", "last_contacted_at", "last_contact_channel", "last_contact_summary", "interaction_count", "qd_score", "next_action"],
"oracle_top_interested_clients": ["name", "phone", "interest_count", "projects", "last_interest_at", "qd_score"],
"oracle_client_360_summary": ["name", "phone", "lead_status", "budget_band", "urgency", "qd_score", "interest_count", "interaction_count", "projects"],
}
defaults: dict[str, dict[str, Any]] = {
"bar_chart": {"xAxis": "category", "yAxis": "value", "sort": "desc", "showLabels": True, "legend": False},
@@ -847,7 +1057,7 @@ class PromptOrchestrator:
Uses the shared runtime LLM service to propose a retrieval plan.
Raises on malformed output so the orchestrator can fall back safely.
"""
row_limit = 50 if ctx.actor_role in ("senior_broker", "junior_broker") else 200
row_limit = _parse_prompt_row_limit(prompt, ctx.actor_role)
system_prompt = (
"You are the Oracle planner for Project Velocity. "
"Return JSON only. "
@@ -855,7 +1065,9 @@ class PromptOrchestrator:
"Allowed component types: pipeline_board, bar_chart, geo_map, table, line_chart, kpi_tile, activity_stream. "
"Allowed datasets: deals, lead_daily_snapshot, lead_geo_interest_rollup, broker_performance, inventory_absorption, "
"oracle_aggregated_metric, lead_activity_log, crm_contacts_overview, crm_opportunity_pipeline, "
"crm_property_interest_rollup, crm_interaction_timeline. "
"crm_property_interest_rollup, crm_interaction_timeline, crm_last_interacted_clients, crm_top_interested_clients, "
"oracle_property_interest_rollup, oracle_client_interaction_timeline, oracle_last_contacted_clients, "
"oracle_top_interested_clients, oracle_client_360_summary. "
"Return an object with keys semanticModelVersion, intentClass, components. "
"Each component must include suggestedType, dataset, and titleHint. "
"Do not emit SQL. Do not invent datasets outside the allowlist."

30
backend/oracle/schema_oracle.sql
View File

@@ -50,6 +50,7 @@ CREATE TABLE IF NOT EXISTS oracle_canvas_components (
version INTEGER NOT NULL DEFAULT 1,
lifecycle_state TEXT NOT NULL DEFAULT 'active' CHECK (lifecycle_state IN ('draft','active','superseded','archived','revoked')),
data_source_descriptor JSONB NOT NULL,
data_rows JSONB NOT NULL DEFAULT '[]'::JSONB,
visualization_parameters JSONB NOT NULL DEFAULT '{}'::JSONB,
data_bindings JSONB NOT NULL DEFAULT '{}'::JSONB,
provenance JSONB NOT NULL,
@@ -63,6 +64,35 @@ CREATE TABLE IF NOT EXISTS oracle_canvas_components (
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
ALTER TABLE oracle_canvas_components
ADD COLUMN IF NOT EXISTS data_rows JSONB NOT NULL DEFAULT '[]'::JSONB;
WITH latest_revisions AS (
SELECT DISTINCT ON (page_id, tenant_id)
page_id,
tenant_id,
components_snapshot
FROM oracle_canvas_page_revisions
ORDER BY page_id, tenant_id, revision_number DESC
),
snapshot_components AS (
SELECT
latest_revisions.page_id,
latest_revisions.tenant_id,
component->>'componentId' AS component_id,
COALESCE(component->'dataRows', '[]'::jsonb) AS data_rows
FROM latest_revisions,
jsonb_array_elements(latest_revisions.components_snapshot) AS component
)
UPDATE oracle_canvas_components occ
SET data_rows = snapshot_components.data_rows
FROM snapshot_components
WHERE occ.page_id = snapshot_components.page_id
AND occ.tenant_id = snapshot_components.tenant_id
AND occ.component_id::text = snapshot_components.component_id
AND occ.data_rows = '[]'::jsonb
AND snapshot_components.data_rows <> '[]'::jsonb;
CREATE TABLE IF NOT EXISTS oracle_prompt_executions (
execution_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id TEXT NOT NULL,

1039
backend/scripts/seed_synthetic_crm.py
View File

File diff suppressed because it is too large Load Diff