From b6f265588d7f50d9142a2c36a17ce0132b69ceeb Mon Sep 17 00:00:00 2001 From: leubeem Date: Mon, 29 Jun 2026 10:27:57 +0200 Subject: [PATCH] feat(pm): LLM routing endpoint, usage capture, and admin review view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend counterpart to the Python engine's semantic router. - POST /api/decision/route: service-secret-guarded endpoint the Python backend calls on regex-miss. Calls gpt-5-mini (ROUTER_LLM_MODEL), validates the chosen id against the candidate set, and writes both a UsageEvent (central cost ledger) and a routing-review record — including timeouts, with timeoutMs + actual latencyMs — so the budget can be tuned - LlmRoutingDecision model + GET /api/admin/llm-routing (paginated, status-filtered, per-status counts) - admin "LLM Routing" tab: transcript vs expected phrase, candidate chips with the chosen one highlighted, latency/budget chip, model reason - serviceAuth util (mirrors CRON_SECRET pattern) - .env.example: ROUTER_LLM_MODEL, SERVICE_SECRET Co-Authored-By: Claude Opus 4.8 --- .env.example | 9 + app/pages/admin/index.vue | 196 +++++++++++++++++++++- server/api/admin/llm-routing/index.get.ts | 64 +++++++ server/api/decision/route.post.ts | 175 +++++++++++++++++++ server/models/LlmRoutingDecision.ts | 80 +++++++++ server/utils/serviceAuth.ts | 32 ++++ 6 files changed, 555 insertions(+), 1 deletion(-) create mode 100644 server/api/admin/llm-routing/index.get.ts create mode 100644 server/api/decision/route.post.ts create mode 100644 server/models/LlmRoutingDecision.ts create mode 100644 server/utils/serviceAuth.ts diff --git a/.env.example b/.env.example index 7c93214..523fae8 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,15 @@ OPENAI_BASE_URL= LLM_MODEL=gpt-5-nano TTS_MODEL=tts-1 VOICE_ID=alloy +# Model used by the /api/decision/route LLM router (the Python backend calls +# this when regex routing misses a pilot transmission). +ROUTER_LLM_MODEL=gpt-5-mini + +# Internal service-to-service auth. The Python decision backend calls +# /api/decision/route with this value in the `x-service-secret` header. Must +# match SERVICE_SECRET in the Python backend's env. If unset, the LLM router is +# disabled (the backend falls back to deterministic bad_next routing). +SERVICE_SECRET=CHANGE_ME # ATC audio generation ATC_OUT_DIR=./storage/atc diff --git a/app/pages/admin/index.vue b/app/pages/admin/index.vue index 269de32..277b79e 100644 --- a/app/pages/admin/index.vue +++ b/app/pages/admin/index.vue @@ -43,6 +43,7 @@ Invitations Waitlist Transmissions + LLM Routing Bug Reports @@ -937,6 +938,115 @@ + +
+
+
+

LLM Routing

+

+ Fälle, in denen die Regex-Routing fehlschlug und das LLM entscheiden musste. + Latenz vs. Zeitlimit hilft beim Tunen des Timeouts. +

+
+
+ + + Laden + +
+
+ + + {{ llmError }} + + +
+ +

LLM-Entscheidungen laden…

+
+ +
+
+ + +
+ {{ d.status }} + + {{ d.latencyMs }}ms / {{ d.timeoutMs }}ms + + {{ d.model }} + ${{ d.costUsd.toFixed(5) }} + {{ formatRelative(d.createdAt) }} +
+ +
+
+

Transkript (STT)

+

"{{ d.transcript }}"

+
+
+

Erwartet

+

"{{ d.expectedPhrase }}"

+
+
+ +
+

Kandidaten

+
+ + mdi-check + {{ c.id }} · {{ c.kind }} + +
+
+ +

„{{ d.reason }}“

+ +
+ State: {{ d.stateId }} · Flow: {{ d.flowSlug || '—' }} · Session: {{ d.sessionId.slice(0, 8) }} +
+ + +
+ +

Keine LLM-Entscheidungen vorhanden.

+ +
+
+ Seite {{ llmPagination.page }} von {{ llmPagination.pages }} · {{ llmPagination.total }} Einträge +
+
+ Zurück + Weiter +
+
+
+
+ +
@@ -1449,7 +1559,7 @@ useHead({ title: 'Admin • OpenSquawk' }) const auth = useAuthStore() const api = useApi() -const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'bug-reports'>('overview') +const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'llm-routing' | 'bug-reports'>('overview') const refreshing = ref(false) const overview = ref(null) @@ -2053,6 +2163,8 @@ watch(activeTab, (tab) => { fetchWaitlist(true) } else if (tab === 'logs' && !sessionsLoaded.value) { fetchSessions(true) + } else if (tab === 'llm-routing' && !llmLoaded.value) { + fetchLlmRouting(true) } else if (tab === 'bug-reports' && !bugReportsLoaded.value) { fetchBugReports(true) } @@ -2145,6 +2257,88 @@ function changeBugReportPage(page: number) { watch(bugReportStatusFilter, () => fetchBugReports(true)) // ──────────────────────────────────────────────────────────────────────────── +// ── LLM Routing ────────────────────────────────────────────────────────────── +interface LlmRoutingItem { + id: string + sessionId: string + flowSlug?: string + stateId: string + transcript: string + expectedPhrase?: string + candidates: { id: string; label?: string; kind: 'ok' | 'bad' }[] + chosen: string | null + reason?: string + status: 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error' + model: string + timeoutMs: number + latencyMs: number + costUsd?: number + createdAt: string | null +} + +interface LlmRoutingResponse { + items: LlmRoutingItem[] + counts: Record + pagination: { total: number; page: number; pageSize: number; pages: number } +} + +const llmDecisions = ref([]) +const llmPagination = reactive({ total: 0, page: 1, pages: 1, pageSize: 20 }) +const llmLoading = ref(false) +const llmError = ref('') +const llmLoaded = ref(false) +const llmStatusFilter = ref<'all' | 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'>('all') +const llmCounts = ref>({}) + +const llmStatusItems = computed(() => [ + { title: `Alle${llmCounts.value.all != null ? ` (${llmCounts.value.all})` : ''}`, value: 'all' }, + { title: `Entschieden${llmCounts.value.decided != null ? ` (${llmCounts.value.decided})` : ''}`, value: 'decided' }, + { title: `Abstain${llmCounts.value.abstain != null ? ` (${llmCounts.value.abstain})` : ''}`, value: 'abstain' }, + { title: `Invalid${llmCounts.value.invalid != null ? ` (${llmCounts.value.invalid})` : ''}`, value: 'invalid' }, + { title: `Timeout${llmCounts.value.timeout != null ? ` (${llmCounts.value.timeout})` : ''}`, value: 'timeout' }, + { title: `Error${llmCounts.value.error != null ? ` (${llmCounts.value.error})` : ''}`, value: 'error' }, +]) + +function llmStatusColor(status: string) { + switch (status) { + case 'decided': return 'green' + case 'abstain': return 'grey' + case 'invalid': return 'orange' + case 'timeout': return 'red' + default: return 'red' + } +} + +async function fetchLlmRouting(resetPage = false) { + if (resetPage) llmPagination.page = 1 + llmLoading.value = true + llmError.value = '' + try { + const response = await api.get('/api/admin/llm-routing', { + query: { status: llmStatusFilter.value, page: llmPagination.page }, + }) + llmDecisions.value = response.items + Object.assign(llmPagination, response.pagination) + const counts = { ...response.counts } + counts.all = Object.values(response.counts).reduce((a, b) => a + b, 0) + llmCounts.value = counts + llmLoaded.value = true + } catch (error) { + llmError.value = extractErrorMessage(error, 'LLM-Entscheidungen konnten nicht geladen werden.') + } finally { + llmLoading.value = false + } +} + +function changeLlmPage(page: number) { + if (page < 1 || page > llmPagination.pages) return + llmPagination.page = page + fetchLlmRouting() +} + +watch(llmStatusFilter, () => fetchLlmRouting(true)) +// ──────────────────────────────────────────────────────────────────────────── + onMounted(() => { loadOverview(true) // Load open bug report count for the badge diff --git a/server/api/admin/llm-routing/index.get.ts b/server/api/admin/llm-routing/index.get.ts new file mode 100644 index 0000000..8a6e7d0 --- /dev/null +++ b/server/api/admin/llm-routing/index.get.ts @@ -0,0 +1,64 @@ +import { defineEventHandler, getQuery } from 'h3' +import { requireAdmin } from '../../../utils/auth' +import { LlmRoutingDecision } from '../../../models/LlmRoutingDecision' + +const STATUSES = ['decided', 'abstain', 'invalid', 'timeout', 'error'] as const + +export default defineEventHandler(async (event) => { + await requireAdmin(event) + + const query = getQuery(event) + const page = Math.max(parseInt(String(query.page ?? '1'), 10) || 1, 1) + const pageSize = 20 + const skip = (page - 1) * pageSize + const status = String(query.status ?? 'all') + + const filter: Record = {} + if ((STATUSES as readonly string[]).includes(status)) { + filter.status = status + } + + const [docs, total, statusCounts] = await Promise.all([ + LlmRoutingDecision.find(filter) + .sort({ createdAt: -1 }) + .skip(skip) + .limit(pageSize) + .lean() + .exec(), + LlmRoutingDecision.countDocuments(filter), + LlmRoutingDecision.aggregate([ + { $group: { _id: '$status', count: { $sum: 1 } } }, + ]), + ]) + + const counts = Object.fromEntries(statusCounts.map((s: any) => [s._id, s.count])) + + const items = docs.map((doc: any) => ({ + id: String(doc._id), + sessionId: doc.sessionId, + flowSlug: doc.flowSlug, + stateId: doc.stateId, + transcript: doc.transcript, + expectedPhrase: doc.expectedPhrase, + candidates: (doc.candidates || []).map((c: any) => ({ id: c.id, label: c.label, kind: c.kind })), + chosen: doc.chosen ?? null, + reason: doc.reason, + status: doc.status, + model: doc.model, + timeoutMs: doc.timeoutMs, + latencyMs: doc.latencyMs, + costUsd: doc.costUsd, + createdAt: doc.createdAt ? new Date(doc.createdAt).toISOString() : null, + })) + + return { + items, + counts, + pagination: { + total, + page, + pageSize, + pages: Math.ceil(total / pageSize) || 1, + }, + } +}) diff --git a/server/api/decision/route.post.ts b/server/api/decision/route.post.ts new file mode 100644 index 0000000..a19d451 --- /dev/null +++ b/server/api/decision/route.post.ts @@ -0,0 +1,175 @@ +// server/api/decision/route.post.ts +// +// Internal endpoint the Python decision backend calls when regex routing fails +// to match a pilot transmission. It asks the LLM to pick the best candidate +// transition for the (often STT-garbled) transcript, records the cost in the +// central usage ledger, and persists a routing-review record — including +// timeouts — so the time budget can be tuned against real latency. + +import { createError, readBody } from 'h3' +import { getOpenAIClient } from '../../utils/openai' +import { requireServiceSecret } from '../../utils/serviceAuth' +import { recordUsage, estimateCostUsd } from '../../utils/usage' +import { + LlmRoutingDecision, + type LlmRoutingCandidate, + type LlmRoutingStatus, +} from '../../models/LlmRoutingDecision' + +interface RouteRequestBody { + sessionId: string + flowSlug?: string + stateId: string + transcript: string + expectedPhrase?: string + candidates: LlmRoutingCandidate[] + timeoutMs?: number +} + +const ROUTER_MODEL = (process.env.ROUTER_LLM_MODEL || 'gpt-5-mini').trim() +const DEFAULT_TIMEOUT_MS = 10_000 + +const SYSTEM_PROMPT = [ + 'You are a routing classifier for an ATC radio-communication trainer.', + 'A pilot transmission was transcribed by speech-to-text and a deterministic regex layer could NOT match it to a next state.', + 'The transcript is frequently garbled by STT errors (split words, wrong numbers, homophones).', + 'Compare the transcript against the expected pilot phrase and choose the single candidate transition whose intent best matches what the pilot most likely said.', + 'Candidates are marked kind="ok" (a correct/expected radio call) or kind="bad" (an incorrect or incomplete call).', + 'Only choose a candidate id from the provided list. If none plausibly matches, choose "none".', + 'Respond with strict JSON only: {"chosen": "", "reason": ""}.', +].join(' ') + +function buildUserPrompt(body: RouteRequestBody): string { + const lines: string[] = [] + lines.push(`Current state: ${body.stateId}${body.flowSlug ? ` (flow ${body.flowSlug})` : ''}`) + if (body.expectedPhrase) { + lines.push(`Expected pilot phrase: "${body.expectedPhrase}"`) + } + lines.push(`Transcript (from STT): "${body.transcript}"`) + lines.push('Candidates:') + for (const c of body.candidates) { + lines.push(`- id="${c.id}" kind=${c.kind}${c.label ? ` label="${c.label}"` : ''}`) + } + return lines.join('\n') +} + +export default defineEventHandler(async (event) => { + requireServiceSecret(event) + + const body = await readBody(event) + if (!body?.transcript || !body?.stateId || !Array.isArray(body?.candidates) || body.candidates.length === 0) { + throw createError({ statusCode: 400, statusMessage: 'transcript, stateId and candidates[] are required.' }) + } + + const timeoutMs = Number.isFinite(body.timeoutMs) && (body.timeoutMs as number) > 0 + ? (body.timeoutMs as number) + : DEFAULT_TIMEOUT_MS + + const candidateIds = new Set(body.candidates.map((c) => c.id)) + const client = getOpenAIClient() + + let status: LlmRoutingStatus = 'error' + let chosen: string | null = null + let reason: string | undefined + let inputTokens: number | undefined + let outputTokens: number | undefined + let costUsd: number | undefined + + const started = Date.now() + try { + const response = await client.chat.completions.create( + { + model: ROUTER_MODEL, + n: 1, + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: buildUserPrompt(body) }, + ], + }, + // Per-request budget; no retries so a slow call fails fast within budget. + { timeout: timeoutMs, maxRetries: 0 }, + ) + + inputTokens = response.usage?.prompt_tokens + outputTokens = response.usage?.completion_tokens + + const raw = response.choices?.[0]?.message?.content?.trim() || '' + let parsedChosen: string | null = null + try { + const parsed = JSON.parse(raw) as { chosen?: string; reason?: string } + parsedChosen = typeof parsed.chosen === 'string' ? parsed.chosen.trim() : null + reason = typeof parsed.reason === 'string' ? parsed.reason.trim() : undefined + } catch { + reason = `Unparseable model output: ${raw.slice(0, 200)}` + } + + if (parsedChosen && candidateIds.has(parsedChosen)) { + chosen = parsedChosen + status = 'decided' + } else if (parsedChosen && parsedChosen.toLowerCase() === 'none') { + status = 'abstain' + } else if (parsedChosen) { + // Model named a state outside the allowed set — never trust it. + status = 'invalid' + reason = reason || `Model returned out-of-set candidate "${parsedChosen}"` + } else { + status = 'error' + } + } catch (err: any) { + const name = String(err?.name || '') + const isTimeout = name.includes('Timeout') || err?.code === 'ETIMEDOUT' || name === 'APIConnectionTimeoutError' + status = isTimeout ? 'timeout' : 'error' + reason = `${name || 'LLM call failed'}: ${String(err?.message || err).slice(0, 200)}` + } + + const latencyMs = Date.now() - started + + // Cost only when the call actually returned token usage (not on timeout/error). + if (inputTokens != null || outputTokens != null) { + costUsd = estimateCostUsd({ + kind: 'llm', + provider: 'openai', + model: ROUTER_MODEL, + endpoint: '/api/decision/route', + inputTokens, + outputTokens, + }) + // Fire-and-forget into the central usage ledger (attributed by sessionId; + // backend has no user id on the runtime session yet). + await recordUsage({ + sessionId: body.sessionId, + kind: 'llm', + provider: 'openai', + model: ROUTER_MODEL, + endpoint: '/api/decision/route', + inputTokens, + outputTokens, + }) + } + + // Always persist the routing-review record, including timeouts/errors. + try { + await LlmRoutingDecision.create({ + sessionId: body.sessionId, + flowSlug: body.flowSlug, + stateId: body.stateId, + transcript: body.transcript, + expectedPhrase: body.expectedPhrase, + candidates: body.candidates, + chosen, + reason, + status, + model: ROUTER_MODEL, + timeoutMs, + latencyMs, + inputTokens, + outputTokens, + costUsd, + }) + } catch (e) { + console.warn('[decision/route] persisting routing decision failed', e) + } + + return { chosen, reason, status, latencyMs, timeoutMs, model: ROUTER_MODEL } +}) diff --git a/server/models/LlmRoutingDecision.ts b/server/models/LlmRoutingDecision.ts new file mode 100644 index 0000000..b1e476b --- /dev/null +++ b/server/models/LlmRoutingDecision.ts @@ -0,0 +1,80 @@ +import mongoose from 'mongoose' + +const { Schema } = mongoose + +// Outcome of a single LLM routing attempt. `decided` = the model picked a valid +// candidate; `abstain` = it returned "none"; `invalid` = it returned something +// outside the candidate set; `timeout` = the request hit the time limit; +// `error` = transport/parse failure. Timeouts and errors are persisted too so +// the configured time limit can be tuned against real latency. +export type LlmRoutingStatus = 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error' + +export interface LlmRoutingCandidate { + id: string + label?: string + kind: 'ok' | 'bad' +} + +export interface LlmRoutingDecisionAttrs { + /** Python runtime session id (no user is known backend-side yet). */ + sessionId: string + flowSlug?: string + stateId: string + /** Raw STT transcript the regex layer failed to route. */ + transcript: string + /** Rendered expected pilot phrase the model compares the transcript against. */ + expectedPhrase?: string + candidates: LlmRoutingCandidate[] + /** Chosen candidate id, or null when the model abstained / failed. */ + chosen: string | null + reason?: string + status: LlmRoutingStatus + model: string + /** Time budget the call was given, in ms (what to tune). */ + timeoutMs: number + /** Wall-clock time the call actually took, in ms (incl. timeouts/errors). */ + latencyMs: number + inputTokens?: number + outputTokens?: number + costUsd?: number + createdAt: Date +} + +const candidateSchema = new Schema( + { + id: { type: String, required: true }, + label: { type: String }, + kind: { type: String, enum: ['ok', 'bad'], required: true }, + }, + { _id: false }, +) + +const llmRoutingDecisionSchema = new mongoose.Schema({ + sessionId: { type: String, required: true, index: true }, + flowSlug: { type: String }, + stateId: { type: String, required: true }, + transcript: { type: String, required: true }, + expectedPhrase: { type: String }, + candidates: { type: [candidateSchema], default: [] }, + chosen: { type: String, default: null }, + reason: { type: String }, + status: { + type: String, + enum: ['decided', 'abstain', 'invalid', 'timeout', 'error'], + required: true, + index: true, + }, + model: { type: String, required: true }, + timeoutMs: { type: Number, required: true }, + latencyMs: { type: Number, required: true }, + inputTokens: { type: Number }, + outputTokens: { type: Number }, + costUsd: { type: Number }, + createdAt: { type: Date, default: () => new Date(), index: true }, +}) + +llmRoutingDecisionSchema.index({ status: 1, createdAt: -1 }) + +export const LlmRoutingDecision = + (mongoose.models.LlmRoutingDecision as mongoose.Model | undefined) || + mongoose.model('LlmRoutingDecision', llmRoutingDecisionSchema) diff --git a/server/utils/serviceAuth.ts b/server/utils/serviceAuth.ts new file mode 100644 index 0000000..a518ca1 --- /dev/null +++ b/server/utils/serviceAuth.ts @@ -0,0 +1,32 @@ +import type { H3Event } from 'h3' +import { createError, getHeader } from 'h3' + +let warnedMissingSecret = false + +/** + * Guards internal server-to-server endpoints (e.g. the Python decision backend + * calling back into Nuxt). The secret is read from SERVICE_SECRET and must be + * supplied via the `x-service-secret` header. + * + * Fail closed: if no secret is configured the endpoint refuses to run (503) + * rather than being publicly callable. + */ +export function requireServiceSecret(event: H3Event) { + const secret = (process.env.SERVICE_SECRET || '').trim() + + if (!secret) { + if (!warnedMissingSecret) { + console.error( + '[service] SERVICE_SECRET is not set — internal endpoints are disabled (503). ' + + 'Set SERVICE_SECRET and pass it via the x-service-secret header.', + ) + warnedMissingSecret = true + } + throw createError({ statusCode: 503, statusMessage: 'Service endpoint is not configured.' }) + } + + const provided = (getHeader(event, 'x-service-secret') || '').trim() + if (provided !== secret) { + throw createError({ statusCode: 401, statusMessage: 'Invalid service secret.' }) + } +}