From 1d09a9fc2f84eafa594cd48be1c63fe5f930156a Mon Sep 17 00:00:00 2001 From: leubeem Date: Mon, 29 Jun 2026 10:27:57 +0200 Subject: [PATCH] feat(pm): LLM routing endpoint, usage capture, and admin review view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend counterpart to the Python engine's semantic router. - POST /api/decision/route: service-secret-guarded endpoint the Python backend calls on regex-miss. Calls gpt-5-mini (ROUTER_LLM_MODEL), validates the chosen id against the candidate set, and writes both a UsageEvent (central cost ledger) and a routing-review record — including timeouts, with timeoutMs + actual latencyMs — so the budget can be tuned - LlmRoutingDecision model + GET /api/admin/llm-routing (paginated, status-filtered, per-status counts) - admin "LLM Routing" tab: transcript vs expected phrase, candidate chips with the chosen one highlighted, latency/budget chip, model reason - serviceAuth util (mirrors CRON_SECRET pattern) - .env.example: ROUTER_LLM_MODEL, SERVICE_SECRET Co-Authored-By: Claude Opus 4.8 --- .env.example | 9 ++ server/api/decision/route.post.ts | 175 ++++++++++++++++++++++++++++ server/models/LlmRoutingDecision.ts | 80 +++++++++++++ server/utils/serviceAuth.ts | 32 +++++ 4 files changed, 296 insertions(+) create mode 100644 server/api/decision/route.post.ts create mode 100644 server/models/LlmRoutingDecision.ts create mode 100644 server/utils/serviceAuth.ts diff --git a/.env.example b/.env.example index 7c93214..523fae8 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,15 @@ OPENAI_BASE_URL= LLM_MODEL=gpt-5-nano TTS_MODEL=tts-1 VOICE_ID=alloy +# Model used by the /api/decision/route LLM router (the Python backend calls +# this when regex routing misses a pilot transmission). +ROUTER_LLM_MODEL=gpt-5-mini + +# Internal service-to-service auth. The Python decision backend calls +# /api/decision/route with this value in the `x-service-secret` header. Must +# match SERVICE_SECRET in the Python backend's env. If unset, the LLM router is +# disabled (the backend falls back to deterministic bad_next routing). +SERVICE_SECRET=CHANGE_ME # ATC audio generation ATC_OUT_DIR=./storage/atc diff --git a/server/api/decision/route.post.ts b/server/api/decision/route.post.ts new file mode 100644 index 0000000..a19d451 --- /dev/null +++ b/server/api/decision/route.post.ts @@ -0,0 +1,175 @@ +// server/api/decision/route.post.ts +// +// Internal endpoint the Python decision backend calls when regex routing fails +// to match a pilot transmission. It asks the LLM to pick the best candidate +// transition for the (often STT-garbled) transcript, records the cost in the +// central usage ledger, and persists a routing-review record — including +// timeouts — so the time budget can be tuned against real latency. + +import { createError, readBody } from 'h3' +import { getOpenAIClient } from '../../utils/openai' +import { requireServiceSecret } from '../../utils/serviceAuth' +import { recordUsage, estimateCostUsd } from '../../utils/usage' +import { + LlmRoutingDecision, + type LlmRoutingCandidate, + type LlmRoutingStatus, +} from '../../models/LlmRoutingDecision' + +interface RouteRequestBody { + sessionId: string + flowSlug?: string + stateId: string + transcript: string + expectedPhrase?: string + candidates: LlmRoutingCandidate[] + timeoutMs?: number +} + +const ROUTER_MODEL = (process.env.ROUTER_LLM_MODEL || 'gpt-5-mini').trim() +const DEFAULT_TIMEOUT_MS = 10_000 + +const SYSTEM_PROMPT = [ + 'You are a routing classifier for an ATC radio-communication trainer.', + 'A pilot transmission was transcribed by speech-to-text and a deterministic regex layer could NOT match it to a next state.', + 'The transcript is frequently garbled by STT errors (split words, wrong numbers, homophones).', + 'Compare the transcript against the expected pilot phrase and choose the single candidate transition whose intent best matches what the pilot most likely said.', + 'Candidates are marked kind="ok" (a correct/expected radio call) or kind="bad" (an incorrect or incomplete call).', + 'Only choose a candidate id from the provided list. If none plausibly matches, choose "none".', + 'Respond with strict JSON only: {"chosen": "", "reason": ""}.', +].join(' ') + +function buildUserPrompt(body: RouteRequestBody): string { + const lines: string[] = [] + lines.push(`Current state: ${body.stateId}${body.flowSlug ? ` (flow ${body.flowSlug})` : ''}`) + if (body.expectedPhrase) { + lines.push(`Expected pilot phrase: "${body.expectedPhrase}"`) + } + lines.push(`Transcript (from STT): "${body.transcript}"`) + lines.push('Candidates:') + for (const c of body.candidates) { + lines.push(`- id="${c.id}" kind=${c.kind}${c.label ? ` label="${c.label}"` : ''}`) + } + return lines.join('\n') +} + +export default defineEventHandler(async (event) => { + requireServiceSecret(event) + + const body = await readBody(event) + if (!body?.transcript || !body?.stateId || !Array.isArray(body?.candidates) || body.candidates.length === 0) { + throw createError({ statusCode: 400, statusMessage: 'transcript, stateId and candidates[] are required.' }) + } + + const timeoutMs = Number.isFinite(body.timeoutMs) && (body.timeoutMs as number) > 0 + ? (body.timeoutMs as number) + : DEFAULT_TIMEOUT_MS + + const candidateIds = new Set(body.candidates.map((c) => c.id)) + const client = getOpenAIClient() + + let status: LlmRoutingStatus = 'error' + let chosen: string | null = null + let reason: string | undefined + let inputTokens: number | undefined + let outputTokens: number | undefined + let costUsd: number | undefined + + const started = Date.now() + try { + const response = await client.chat.completions.create( + { + model: ROUTER_MODEL, + n: 1, + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: buildUserPrompt(body) }, + ], + }, + // Per-request budget; no retries so a slow call fails fast within budget. + { timeout: timeoutMs, maxRetries: 0 }, + ) + + inputTokens = response.usage?.prompt_tokens + outputTokens = response.usage?.completion_tokens + + const raw = response.choices?.[0]?.message?.content?.trim() || '' + let parsedChosen: string | null = null + try { + const parsed = JSON.parse(raw) as { chosen?: string; reason?: string } + parsedChosen = typeof parsed.chosen === 'string' ? parsed.chosen.trim() : null + reason = typeof parsed.reason === 'string' ? parsed.reason.trim() : undefined + } catch { + reason = `Unparseable model output: ${raw.slice(0, 200)}` + } + + if (parsedChosen && candidateIds.has(parsedChosen)) { + chosen = parsedChosen + status = 'decided' + } else if (parsedChosen && parsedChosen.toLowerCase() === 'none') { + status = 'abstain' + } else if (parsedChosen) { + // Model named a state outside the allowed set — never trust it. + status = 'invalid' + reason = reason || `Model returned out-of-set candidate "${parsedChosen}"` + } else { + status = 'error' + } + } catch (err: any) { + const name = String(err?.name || '') + const isTimeout = name.includes('Timeout') || err?.code === 'ETIMEDOUT' || name === 'APIConnectionTimeoutError' + status = isTimeout ? 'timeout' : 'error' + reason = `${name || 'LLM call failed'}: ${String(err?.message || err).slice(0, 200)}` + } + + const latencyMs = Date.now() - started + + // Cost only when the call actually returned token usage (not on timeout/error). + if (inputTokens != null || outputTokens != null) { + costUsd = estimateCostUsd({ + kind: 'llm', + provider: 'openai', + model: ROUTER_MODEL, + endpoint: '/api/decision/route', + inputTokens, + outputTokens, + }) + // Fire-and-forget into the central usage ledger (attributed by sessionId; + // backend has no user id on the runtime session yet). + await recordUsage({ + sessionId: body.sessionId, + kind: 'llm', + provider: 'openai', + model: ROUTER_MODEL, + endpoint: '/api/decision/route', + inputTokens, + outputTokens, + }) + } + + // Always persist the routing-review record, including timeouts/errors. + try { + await LlmRoutingDecision.create({ + sessionId: body.sessionId, + flowSlug: body.flowSlug, + stateId: body.stateId, + transcript: body.transcript, + expectedPhrase: body.expectedPhrase, + candidates: body.candidates, + chosen, + reason, + status, + model: ROUTER_MODEL, + timeoutMs, + latencyMs, + inputTokens, + outputTokens, + costUsd, + }) + } catch (e) { + console.warn('[decision/route] persisting routing decision failed', e) + } + + return { chosen, reason, status, latencyMs, timeoutMs, model: ROUTER_MODEL } +}) diff --git a/server/models/LlmRoutingDecision.ts b/server/models/LlmRoutingDecision.ts new file mode 100644 index 0000000..b1e476b --- /dev/null +++ b/server/models/LlmRoutingDecision.ts @@ -0,0 +1,80 @@ +import mongoose from 'mongoose' + +const { Schema } = mongoose + +// Outcome of a single LLM routing attempt. `decided` = the model picked a valid +// candidate; `abstain` = it returned "none"; `invalid` = it returned something +// outside the candidate set; `timeout` = the request hit the time limit; +// `error` = transport/parse failure. Timeouts and errors are persisted too so +// the configured time limit can be tuned against real latency. +export type LlmRoutingStatus = 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error' + +export interface LlmRoutingCandidate { + id: string + label?: string + kind: 'ok' | 'bad' +} + +export interface LlmRoutingDecisionAttrs { + /** Python runtime session id (no user is known backend-side yet). */ + sessionId: string + flowSlug?: string + stateId: string + /** Raw STT transcript the regex layer failed to route. */ + transcript: string + /** Rendered expected pilot phrase the model compares the transcript against. */ + expectedPhrase?: string + candidates: LlmRoutingCandidate[] + /** Chosen candidate id, or null when the model abstained / failed. */ + chosen: string | null + reason?: string + status: LlmRoutingStatus + model: string + /** Time budget the call was given, in ms (what to tune). */ + timeoutMs: number + /** Wall-clock time the call actually took, in ms (incl. timeouts/errors). */ + latencyMs: number + inputTokens?: number + outputTokens?: number + costUsd?: number + createdAt: Date +} + +const candidateSchema = new Schema( + { + id: { type: String, required: true }, + label: { type: String }, + kind: { type: String, enum: ['ok', 'bad'], required: true }, + }, + { _id: false }, +) + +const llmRoutingDecisionSchema = new mongoose.Schema({ + sessionId: { type: String, required: true, index: true }, + flowSlug: { type: String }, + stateId: { type: String, required: true }, + transcript: { type: String, required: true }, + expectedPhrase: { type: String }, + candidates: { type: [candidateSchema], default: [] }, + chosen: { type: String, default: null }, + reason: { type: String }, + status: { + type: String, + enum: ['decided', 'abstain', 'invalid', 'timeout', 'error'], + required: true, + index: true, + }, + model: { type: String, required: true }, + timeoutMs: { type: Number, required: true }, + latencyMs: { type: Number, required: true }, + inputTokens: { type: Number }, + outputTokens: { type: Number }, + costUsd: { type: Number }, + createdAt: { type: Date, default: () => new Date(), index: true }, +}) + +llmRoutingDecisionSchema.index({ status: 1, createdAt: -1 }) + +export const LlmRoutingDecision = + (mongoose.models.LlmRoutingDecision as mongoose.Model | undefined) || + mongoose.model('LlmRoutingDecision', llmRoutingDecisionSchema) diff --git a/server/utils/serviceAuth.ts b/server/utils/serviceAuth.ts new file mode 100644 index 0000000..a518ca1 --- /dev/null +++ b/server/utils/serviceAuth.ts @@ -0,0 +1,32 @@ +import type { H3Event } from 'h3' +import { createError, getHeader } from 'h3' + +let warnedMissingSecret = false + +/** + * Guards internal server-to-server endpoints (e.g. the Python decision backend + * calling back into Nuxt). The secret is read from SERVICE_SECRET and must be + * supplied via the `x-service-secret` header. + * + * Fail closed: if no secret is configured the endpoint refuses to run (503) + * rather than being publicly callable. + */ +export function requireServiceSecret(event: H3Event) { + const secret = (process.env.SERVICE_SECRET || '').trim() + + if (!secret) { + if (!warnedMissingSecret) { + console.error( + '[service] SERVICE_SECRET is not set — internal endpoints are disabled (503). ' + + 'Set SERVICE_SECRET and pass it via the x-service-secret header.', + ) + warnedMissingSecret = true + } + throw createError({ statusCode: 503, statusMessage: 'Service endpoint is not configured.' }) + } + + const provided = (getHeader(event, 'x-service-secret') || '').trim() + if (provided !== secret) { + throw createError({ statusCode: 401, statusMessage: 'Invalid service secret.' }) + } +}