feat(pm): LLM routing endpoint, usage capture, and admin review view

Backend counterpart to the Python engine's semantic router.

- POST /api/decision/route: service-secret-guarded endpoint the Python
  backend calls on regex-miss. Calls gpt-5-mini (ROUTER_LLM_MODEL),
  validates the chosen id against the candidate set, and writes both a
  UsageEvent (central cost ledger) and a routing-review record — including
  timeouts, with timeoutMs + actual latencyMs — so the budget can be tuned
- LlmRoutingDecision model + GET /api/admin/llm-routing (paginated,
  status-filtered, per-status counts)
- admin "LLM Routing" tab: transcript vs expected phrase, candidate chips
  with the chosen one highlighted, latency/budget chip, model reason
- serviceAuth util (mirrors CRON_SECRET pattern)
- .env.example: ROUTER_LLM_MODEL, SERVICE_SECRET

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
leubeem
2026-06-29 10:27:57 +02:00
parent 77ddaabd4d
commit b6f265588d
6 changed files with 555 additions and 1 deletions

View File

@@ -0,0 +1,64 @@
import { defineEventHandler, getQuery } from 'h3'
import { requireAdmin } from '../../../utils/auth'
import { LlmRoutingDecision } from '../../../models/LlmRoutingDecision'
const STATUSES = ['decided', 'abstain', 'invalid', 'timeout', 'error'] as const
export default defineEventHandler(async (event) => {
await requireAdmin(event)
const query = getQuery(event)
const page = Math.max(parseInt(String(query.page ?? '1'), 10) || 1, 1)
const pageSize = 20
const skip = (page - 1) * pageSize
const status = String(query.status ?? 'all')
const filter: Record<string, any> = {}
if ((STATUSES as readonly string[]).includes(status)) {
filter.status = status
}
const [docs, total, statusCounts] = await Promise.all([
LlmRoutingDecision.find(filter)
.sort({ createdAt: -1 })
.skip(skip)
.limit(pageSize)
.lean()
.exec(),
LlmRoutingDecision.countDocuments(filter),
LlmRoutingDecision.aggregate([
{ $group: { _id: '$status', count: { $sum: 1 } } },
]),
])
const counts = Object.fromEntries(statusCounts.map((s: any) => [s._id, s.count]))
const items = docs.map((doc: any) => ({
id: String(doc._id),
sessionId: doc.sessionId,
flowSlug: doc.flowSlug,
stateId: doc.stateId,
transcript: doc.transcript,
expectedPhrase: doc.expectedPhrase,
candidates: (doc.candidates || []).map((c: any) => ({ id: c.id, label: c.label, kind: c.kind })),
chosen: doc.chosen ?? null,
reason: doc.reason,
status: doc.status,
model: doc.model,
timeoutMs: doc.timeoutMs,
latencyMs: doc.latencyMs,
costUsd: doc.costUsd,
createdAt: doc.createdAt ? new Date(doc.createdAt).toISOString() : null,
}))
return {
items,
counts,
pagination: {
total,
page,
pageSize,
pages: Math.ceil(total / pageSize) || 1,
},
}
})