@@ -1449,7 +1559,7 @@ useHead({ title: 'Admin • OpenSquawk' })
const auth = useAuthStore()
const api = useApi()
-const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'bug-reports'>('overview')
+const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'llm-routing' | 'bug-reports'>('overview')
const refreshing = ref(false)
const overview = ref(null)
@@ -2053,6 +2163,8 @@ watch(activeTab, (tab) => {
fetchWaitlist(true)
} else if (tab === 'logs' && !sessionsLoaded.value) {
fetchSessions(true)
+ } else if (tab === 'llm-routing' && !llmLoaded.value) {
+ fetchLlmRouting(true)
} else if (tab === 'bug-reports' && !bugReportsLoaded.value) {
fetchBugReports(true)
}
@@ -2145,6 +2257,88 @@ function changeBugReportPage(page: number) {
watch(bugReportStatusFilter, () => fetchBugReports(true))
// ────────────────────────────────────────────────────────────────────────────
+// ── LLM Routing ──────────────────────────────────────────────────────────────
+interface LlmRoutingItem {
+ id: string
+ sessionId: string
+ flowSlug?: string
+ stateId: string
+ transcript: string
+ expectedPhrase?: string
+ candidates: { id: string; label?: string; kind: 'ok' | 'bad' }[]
+ chosen: string | null
+ reason?: string
+ status: 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'
+ model: string
+ timeoutMs: number
+ latencyMs: number
+ costUsd?: number
+ createdAt: string | null
+}
+
+interface LlmRoutingResponse {
+ items: LlmRoutingItem[]
+ counts: Record
+ pagination: { total: number; page: number; pageSize: number; pages: number }
+}
+
+const llmDecisions = ref([])
+const llmPagination = reactive({ total: 0, page: 1, pages: 1, pageSize: 20 })
+const llmLoading = ref(false)
+const llmError = ref('')
+const llmLoaded = ref(false)
+const llmStatusFilter = ref<'all' | 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'>('all')
+const llmCounts = ref>({})
+
+const llmStatusItems = computed(() => [
+ { title: `Alle${llmCounts.value.all != null ? ` (${llmCounts.value.all})` : ''}`, value: 'all' },
+ { title: `Entschieden${llmCounts.value.decided != null ? ` (${llmCounts.value.decided})` : ''}`, value: 'decided' },
+ { title: `Abstain${llmCounts.value.abstain != null ? ` (${llmCounts.value.abstain})` : ''}`, value: 'abstain' },
+ { title: `Invalid${llmCounts.value.invalid != null ? ` (${llmCounts.value.invalid})` : ''}`, value: 'invalid' },
+ { title: `Timeout${llmCounts.value.timeout != null ? ` (${llmCounts.value.timeout})` : ''}`, value: 'timeout' },
+ { title: `Error${llmCounts.value.error != null ? ` (${llmCounts.value.error})` : ''}`, value: 'error' },
+])
+
+function llmStatusColor(status: string) {
+ switch (status) {
+ case 'decided': return 'green'
+ case 'abstain': return 'grey'
+ case 'invalid': return 'orange'
+ case 'timeout': return 'red'
+ default: return 'red'
+ }
+}
+
+async function fetchLlmRouting(resetPage = false) {
+ if (resetPage) llmPagination.page = 1
+ llmLoading.value = true
+ llmError.value = ''
+ try {
+ const response = await api.get('/api/admin/llm-routing', {
+ query: { status: llmStatusFilter.value, page: llmPagination.page },
+ })
+ llmDecisions.value = response.items
+ Object.assign(llmPagination, response.pagination)
+ const counts = { ...response.counts }
+ counts.all = Object.values(response.counts).reduce((a, b) => a + b, 0)
+ llmCounts.value = counts
+ llmLoaded.value = true
+ } catch (error) {
+ llmError.value = extractErrorMessage(error, 'LLM-Entscheidungen konnten nicht geladen werden.')
+ } finally {
+ llmLoading.value = false
+ }
+}
+
+function changeLlmPage(page: number) {
+ if (page < 1 || page > llmPagination.pages) return
+ llmPagination.page = page
+ fetchLlmRouting()
+}
+
+watch(llmStatusFilter, () => fetchLlmRouting(true))
+// ────────────────────────────────────────────────────────────────────────────
+
onMounted(() => {
loadOverview(true)
// Load open bug report count for the badge
diff --git a/server/api/admin/llm-routing/index.get.ts b/server/api/admin/llm-routing/index.get.ts
new file mode 100644
index 0000000..8a6e7d0
--- /dev/null
+++ b/server/api/admin/llm-routing/index.get.ts
@@ -0,0 +1,64 @@
+import { defineEventHandler, getQuery } from 'h3'
+import { requireAdmin } from '../../../utils/auth'
+import { LlmRoutingDecision } from '../../../models/LlmRoutingDecision'
+
+const STATUSES = ['decided', 'abstain', 'invalid', 'timeout', 'error'] as const
+
+export default defineEventHandler(async (event) => {
+ await requireAdmin(event)
+
+ const query = getQuery(event)
+ const page = Math.max(parseInt(String(query.page ?? '1'), 10) || 1, 1)
+ const pageSize = 20
+ const skip = (page - 1) * pageSize
+ const status = String(query.status ?? 'all')
+
+ const filter: Record = {}
+ if ((STATUSES as readonly string[]).includes(status)) {
+ filter.status = status
+ }
+
+ const [docs, total, statusCounts] = await Promise.all([
+ LlmRoutingDecision.find(filter)
+ .sort({ createdAt: -1 })
+ .skip(skip)
+ .limit(pageSize)
+ .lean()
+ .exec(),
+ LlmRoutingDecision.countDocuments(filter),
+ LlmRoutingDecision.aggregate([
+ { $group: { _id: '$status', count: { $sum: 1 } } },
+ ]),
+ ])
+
+ const counts = Object.fromEntries(statusCounts.map((s: any) => [s._id, s.count]))
+
+ const items = docs.map((doc: any) => ({
+ id: String(doc._id),
+ sessionId: doc.sessionId,
+ flowSlug: doc.flowSlug,
+ stateId: doc.stateId,
+ transcript: doc.transcript,
+ expectedPhrase: doc.expectedPhrase,
+ candidates: (doc.candidates || []).map((c: any) => ({ id: c.id, label: c.label, kind: c.kind })),
+ chosen: doc.chosen ?? null,
+ reason: doc.reason,
+ status: doc.status,
+ model: doc.model,
+ timeoutMs: doc.timeoutMs,
+ latencyMs: doc.latencyMs,
+ costUsd: doc.costUsd,
+ createdAt: doc.createdAt ? new Date(doc.createdAt).toISOString() : null,
+ }))
+
+ return {
+ items,
+ counts,
+ pagination: {
+ total,
+ page,
+ pageSize,
+ pages: Math.ceil(total / pageSize) || 1,
+ },
+ }
+})
diff --git a/server/api/decision/route.post.ts b/server/api/decision/route.post.ts
new file mode 100644
index 0000000..a19d451
--- /dev/null
+++ b/server/api/decision/route.post.ts
@@ -0,0 +1,175 @@
+// server/api/decision/route.post.ts
+//
+// Internal endpoint the Python decision backend calls when regex routing fails
+// to match a pilot transmission. It asks the LLM to pick the best candidate
+// transition for the (often STT-garbled) transcript, records the cost in the
+// central usage ledger, and persists a routing-review record — including
+// timeouts — so the time budget can be tuned against real latency.
+
+import { createError, readBody } from 'h3'
+import { getOpenAIClient } from '../../utils/openai'
+import { requireServiceSecret } from '../../utils/serviceAuth'
+import { recordUsage, estimateCostUsd } from '../../utils/usage'
+import {
+ LlmRoutingDecision,
+ type LlmRoutingCandidate,
+ type LlmRoutingStatus,
+} from '../../models/LlmRoutingDecision'
+
+interface RouteRequestBody {
+ sessionId: string
+ flowSlug?: string
+ stateId: string
+ transcript: string
+ expectedPhrase?: string
+ candidates: LlmRoutingCandidate[]
+ timeoutMs?: number
+}
+
+const ROUTER_MODEL = (process.env.ROUTER_LLM_MODEL || 'gpt-5-mini').trim()
+const DEFAULT_TIMEOUT_MS = 10_000
+
+const SYSTEM_PROMPT = [
+ 'You are a routing classifier for an ATC radio-communication trainer.',
+ 'A pilot transmission was transcribed by speech-to-text and a deterministic regex layer could NOT match it to a next state.',
+ 'The transcript is frequently garbled by STT errors (split words, wrong numbers, homophones).',
+ 'Compare the transcript against the expected pilot phrase and choose the single candidate transition whose intent best matches what the pilot most likely said.',
+ 'Candidates are marked kind="ok" (a correct/expected radio call) or kind="bad" (an incorrect or incomplete call).',
+ 'Only choose a candidate id from the provided list. If none plausibly matches, choose "none".',
+ 'Respond with strict JSON only: {"chosen": "", "reason": ""}.',
+].join(' ')
+
+function buildUserPrompt(body: RouteRequestBody): string {
+ const lines: string[] = []
+ lines.push(`Current state: ${body.stateId}${body.flowSlug ? ` (flow ${body.flowSlug})` : ''}`)
+ if (body.expectedPhrase) {
+ lines.push(`Expected pilot phrase: "${body.expectedPhrase}"`)
+ }
+ lines.push(`Transcript (from STT): "${body.transcript}"`)
+ lines.push('Candidates:')
+ for (const c of body.candidates) {
+ lines.push(`- id="${c.id}" kind=${c.kind}${c.label ? ` label="${c.label}"` : ''}`)
+ }
+ return lines.join('\n')
+}
+
+export default defineEventHandler(async (event) => {
+ requireServiceSecret(event)
+
+ const body = await readBody(event)
+ if (!body?.transcript || !body?.stateId || !Array.isArray(body?.candidates) || body.candidates.length === 0) {
+ throw createError({ statusCode: 400, statusMessage: 'transcript, stateId and candidates[] are required.' })
+ }
+
+ const timeoutMs = Number.isFinite(body.timeoutMs) && (body.timeoutMs as number) > 0
+ ? (body.timeoutMs as number)
+ : DEFAULT_TIMEOUT_MS
+
+ const candidateIds = new Set(body.candidates.map((c) => c.id))
+ const client = getOpenAIClient()
+
+ let status: LlmRoutingStatus = 'error'
+ let chosen: string | null = null
+ let reason: string | undefined
+ let inputTokens: number | undefined
+ let outputTokens: number | undefined
+ let costUsd: number | undefined
+
+ const started = Date.now()
+ try {
+ const response = await client.chat.completions.create(
+ {
+ model: ROUTER_MODEL,
+ n: 1,
+ response_format: { type: 'json_object' },
+ messages: [
+ { role: 'system', content: SYSTEM_PROMPT },
+ { role: 'user', content: buildUserPrompt(body) },
+ ],
+ },
+ // Per-request budget; no retries so a slow call fails fast within budget.
+ { timeout: timeoutMs, maxRetries: 0 },
+ )
+
+ inputTokens = response.usage?.prompt_tokens
+ outputTokens = response.usage?.completion_tokens
+
+ const raw = response.choices?.[0]?.message?.content?.trim() || ''
+ let parsedChosen: string | null = null
+ try {
+ const parsed = JSON.parse(raw) as { chosen?: string; reason?: string }
+ parsedChosen = typeof parsed.chosen === 'string' ? parsed.chosen.trim() : null
+ reason = typeof parsed.reason === 'string' ? parsed.reason.trim() : undefined
+ } catch {
+ reason = `Unparseable model output: ${raw.slice(0, 200)}`
+ }
+
+ if (parsedChosen && candidateIds.has(parsedChosen)) {
+ chosen = parsedChosen
+ status = 'decided'
+ } else if (parsedChosen && parsedChosen.toLowerCase() === 'none') {
+ status = 'abstain'
+ } else if (parsedChosen) {
+ // Model named a state outside the allowed set — never trust it.
+ status = 'invalid'
+ reason = reason || `Model returned out-of-set candidate "${parsedChosen}"`
+ } else {
+ status = 'error'
+ }
+ } catch (err: any) {
+ const name = String(err?.name || '')
+ const isTimeout = name.includes('Timeout') || err?.code === 'ETIMEDOUT' || name === 'APIConnectionTimeoutError'
+ status = isTimeout ? 'timeout' : 'error'
+ reason = `${name || 'LLM call failed'}: ${String(err?.message || err).slice(0, 200)}`
+ }
+
+ const latencyMs = Date.now() - started
+
+ // Cost only when the call actually returned token usage (not on timeout/error).
+ if (inputTokens != null || outputTokens != null) {
+ costUsd = estimateCostUsd({
+ kind: 'llm',
+ provider: 'openai',
+ model: ROUTER_MODEL,
+ endpoint: '/api/decision/route',
+ inputTokens,
+ outputTokens,
+ })
+ // Fire-and-forget into the central usage ledger (attributed by sessionId;
+ // backend has no user id on the runtime session yet).
+ await recordUsage({
+ sessionId: body.sessionId,
+ kind: 'llm',
+ provider: 'openai',
+ model: ROUTER_MODEL,
+ endpoint: '/api/decision/route',
+ inputTokens,
+ outputTokens,
+ })
+ }
+
+ // Always persist the routing-review record, including timeouts/errors.
+ try {
+ await LlmRoutingDecision.create({
+ sessionId: body.sessionId,
+ flowSlug: body.flowSlug,
+ stateId: body.stateId,
+ transcript: body.transcript,
+ expectedPhrase: body.expectedPhrase,
+ candidates: body.candidates,
+ chosen,
+ reason,
+ status,
+ model: ROUTER_MODEL,
+ timeoutMs,
+ latencyMs,
+ inputTokens,
+ outputTokens,
+ costUsd,
+ })
+ } catch (e) {
+ console.warn('[decision/route] persisting routing decision failed', e)
+ }
+
+ return { chosen, reason, status, latencyMs, timeoutMs, model: ROUTER_MODEL }
+})
diff --git a/server/models/LlmRoutingDecision.ts b/server/models/LlmRoutingDecision.ts
new file mode 100644
index 0000000..b1e476b
--- /dev/null
+++ b/server/models/LlmRoutingDecision.ts
@@ -0,0 +1,80 @@
+import mongoose from 'mongoose'
+
+const { Schema } = mongoose
+
+// Outcome of a single LLM routing attempt. `decided` = the model picked a valid
+// candidate; `abstain` = it returned "none"; `invalid` = it returned something
+// outside the candidate set; `timeout` = the request hit the time limit;
+// `error` = transport/parse failure. Timeouts and errors are persisted too so
+// the configured time limit can be tuned against real latency.
+export type LlmRoutingStatus = 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'
+
+export interface LlmRoutingCandidate {
+ id: string
+ label?: string
+ kind: 'ok' | 'bad'
+}
+
+export interface LlmRoutingDecisionAttrs {
+ /** Python runtime session id (no user is known backend-side yet). */
+ sessionId: string
+ flowSlug?: string
+ stateId: string
+ /** Raw STT transcript the regex layer failed to route. */
+ transcript: string
+ /** Rendered expected pilot phrase the model compares the transcript against. */
+ expectedPhrase?: string
+ candidates: LlmRoutingCandidate[]
+ /** Chosen candidate id, or null when the model abstained / failed. */
+ chosen: string | null
+ reason?: string
+ status: LlmRoutingStatus
+ model: string
+ /** Time budget the call was given, in ms (what to tune). */
+ timeoutMs: number
+ /** Wall-clock time the call actually took, in ms (incl. timeouts/errors). */
+ latencyMs: number
+ inputTokens?: number
+ outputTokens?: number
+ costUsd?: number
+ createdAt: Date
+}
+
+const candidateSchema = new Schema(
+ {
+ id: { type: String, required: true },
+ label: { type: String },
+ kind: { type: String, enum: ['ok', 'bad'], required: true },
+ },
+ { _id: false },
+)
+
+const llmRoutingDecisionSchema = new mongoose.Schema({
+ sessionId: { type: String, required: true, index: true },
+ flowSlug: { type: String },
+ stateId: { type: String, required: true },
+ transcript: { type: String, required: true },
+ expectedPhrase: { type: String },
+ candidates: { type: [candidateSchema], default: [] },
+ chosen: { type: String, default: null },
+ reason: { type: String },
+ status: {
+ type: String,
+ enum: ['decided', 'abstain', 'invalid', 'timeout', 'error'],
+ required: true,
+ index: true,
+ },
+ model: { type: String, required: true },
+ timeoutMs: { type: Number, required: true },
+ latencyMs: { type: Number, required: true },
+ inputTokens: { type: Number },
+ outputTokens: { type: Number },
+ costUsd: { type: Number },
+ createdAt: { type: Date, default: () => new Date(), index: true },
+})
+
+llmRoutingDecisionSchema.index({ status: 1, createdAt: -1 })
+
+export const LlmRoutingDecision =
+ (mongoose.models.LlmRoutingDecision as mongoose.Model | undefined) ||
+ mongoose.model('LlmRoutingDecision', llmRoutingDecisionSchema)
diff --git a/server/utils/serviceAuth.ts b/server/utils/serviceAuth.ts
new file mode 100644
index 0000000..a518ca1
--- /dev/null
+++ b/server/utils/serviceAuth.ts
@@ -0,0 +1,32 @@
+import type { H3Event } from 'h3'
+import { createError, getHeader } from 'h3'
+
+let warnedMissingSecret = false
+
+/**
+ * Guards internal server-to-server endpoints (e.g. the Python decision backend
+ * calling back into Nuxt). The secret is read from SERVICE_SECRET and must be
+ * supplied via the `x-service-secret` header.
+ *
+ * Fail closed: if no secret is configured the endpoint refuses to run (503)
+ * rather than being publicly callable.
+ */
+export function requireServiceSecret(event: H3Event) {
+ const secret = (process.env.SERVICE_SECRET || '').trim()
+
+ if (!secret) {
+ if (!warnedMissingSecret) {
+ console.error(
+ '[service] SERVICE_SECRET is not set — internal endpoints are disabled (503). ' +
+ 'Set SERVICE_SECRET and pass it via the x-service-secret header.',
+ )
+ warnedMissingSecret = true
+ }
+ throw createError({ statusCode: 503, statusMessage: 'Service endpoint is not configured.' })
+ }
+
+ const provided = (getHeader(event, 'x-service-secret') || '').trim()
+ if (provided !== secret) {
+ throw createError({ statusCode: 401, statusMessage: 'Invalid service secret.' })
+ }
+}