mirror of
https://github.com/OpenSquawk/OpenSquawk
synced 2026-08-05 08:55:54 +08:00
feat(pm): LLM routing endpoint, usage capture, and admin review view
Backend counterpart to the Python engine's semantic router. - POST /api/decision/route: service-secret-guarded endpoint the Python backend calls on regex-miss. Calls gpt-5-mini (ROUTER_LLM_MODEL), validates the chosen id against the candidate set, and writes both a UsageEvent (central cost ledger) and a routing-review record — including timeouts, with timeoutMs + actual latencyMs — so the budget can be tuned - LlmRoutingDecision model + GET /api/admin/llm-routing (paginated, status-filtered, per-status counts) - admin "LLM Routing" tab: transcript vs expected phrase, candidate chips with the chosen one highlighted, latency/budget chip, model reason - serviceAuth util (mirrors CRON_SECRET pattern) - .env.example: ROUTER_LLM_MODEL, SERVICE_SECRET Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,15 @@ OPENAI_BASE_URL=
|
||||
LLM_MODEL=gpt-5-nano
|
||||
TTS_MODEL=tts-1
|
||||
VOICE_ID=alloy
|
||||
# Model used by the /api/decision/route LLM router (the Python backend calls
|
||||
# this when regex routing misses a pilot transmission).
|
||||
ROUTER_LLM_MODEL=gpt-5-mini
|
||||
|
||||
# Internal service-to-service auth. The Python decision backend calls
|
||||
# /api/decision/route with this value in the `x-service-secret` header. Must
|
||||
# match SERVICE_SECRET in the Python backend's env. If unset, the LLM router is
|
||||
# disabled (the backend falls back to deterministic bad_next routing).
|
||||
SERVICE_SECRET=CHANGE_ME
|
||||
|
||||
# ATC audio generation
|
||||
ATC_OUT_DIR=./storage/atc
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
<v-tab value="invitations">Invitations</v-tab>
|
||||
<v-tab value="waitlist">Waitlist</v-tab>
|
||||
<v-tab value="logs">Transmissions</v-tab>
|
||||
<v-tab value="llm-routing">LLM Routing</v-tab>
|
||||
<v-tab value="bug-reports">
|
||||
Bug Reports
|
||||
<v-badge v-if="bugReportOpenCount > 0" :content="bugReportOpenCount" color="red" inline class="ml-1" />
|
||||
@@ -937,6 +938,115 @@
|
||||
</v-window-item>
|
||||
|
||||
<!-- Bug Reports Tab -->
|
||||
<v-window-item value="llm-routing">
|
||||
<section class="space-y-6">
|
||||
<div class="flex flex-col gap-4 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
<h2 class="text-2xl font-semibold">LLM Routing</h2>
|
||||
<p class="text-sm text-white/70">
|
||||
Fälle, in denen die Regex-Routing fehlschlug und das LLM entscheiden musste.
|
||||
Latenz vs. Zeitlimit hilft beim Tunen des Timeouts.
|
||||
</p>
|
||||
</div>
|
||||
<div class="flex items-center gap-3">
|
||||
<v-select
|
||||
v-model="llmStatusFilter"
|
||||
:items="llmStatusItems"
|
||||
label="Status"
|
||||
density="comfortable"
|
||||
variant="outlined"
|
||||
color="cyan"
|
||||
hide-details
|
||||
class="w-48"
|
||||
/>
|
||||
<v-btn color="cyan" variant="tonal" :loading="llmLoading" @click="fetchLlmRouting(true)">
|
||||
Laden
|
||||
</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<v-alert v-if="llmError" type="warning" variant="tonal" density="comfortable" class="bg-red-500/10 text-red-100">
|
||||
{{ llmError }}
|
||||
</v-alert>
|
||||
|
||||
<div v-if="llmLoading && !llmDecisions.length" class="py-12 text-center text-white/70">
|
||||
<v-progress-circular indeterminate color="cyan" class="mb-4" />
|
||||
<p>LLM-Entscheidungen laden…</p>
|
||||
</div>
|
||||
|
||||
<div v-else class="space-y-4">
|
||||
<div v-if="llmDecisions.length" class="space-y-4">
|
||||
<v-card
|
||||
v-for="d in llmDecisions"
|
||||
:key="d.id"
|
||||
class="border border-white/10 bg-black/40"
|
||||
>
|
||||
<v-card-text class="space-y-3">
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<v-chip size="x-small" :color="llmStatusColor(d.status)" variant="tonal">{{ d.status }}</v-chip>
|
||||
<v-chip
|
||||
size="x-small"
|
||||
:color="d.latencyMs >= d.timeoutMs ? 'red' : (d.latencyMs > d.timeoutMs * 0.8 ? 'orange' : 'cyan')"
|
||||
variant="tonal"
|
||||
>
|
||||
{{ d.latencyMs }}ms / {{ d.timeoutMs }}ms
|
||||
</v-chip>
|
||||
<span class="text-xs text-white/40 font-mono">{{ d.model }}</span>
|
||||
<span v-if="d.costUsd != null" class="text-xs text-white/40">${{ d.costUsd.toFixed(5) }}</span>
|
||||
<span class="text-xs text-white/40">{{ formatRelative(d.createdAt) }}</span>
|
||||
</div>
|
||||
|
||||
<div class="grid gap-2 sm:grid-cols-2">
|
||||
<div class="space-y-1">
|
||||
<p class="text-[11px] uppercase tracking-wide text-white/40">Transkript (STT)</p>
|
||||
<p class="text-sm text-white/90 font-mono">"{{ d.transcript }}"</p>
|
||||
</div>
|
||||
<div v-if="d.expectedPhrase" class="space-y-1">
|
||||
<p class="text-[11px] uppercase tracking-wide text-white/40">Erwartet</p>
|
||||
<p class="text-sm text-white/70 font-mono">"{{ d.expectedPhrase }}"</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-1">
|
||||
<p class="text-[11px] uppercase tracking-wide text-white/40">Kandidaten</p>
|
||||
<div class="flex flex-wrap gap-1.5">
|
||||
<v-chip
|
||||
v-for="c in d.candidates"
|
||||
:key="c.id"
|
||||
size="x-small"
|
||||
:color="c.id === d.chosen ? 'green' : (c.kind === 'ok' ? 'cyan' : 'grey')"
|
||||
:variant="c.id === d.chosen ? 'flat' : 'outlined'"
|
||||
>
|
||||
<v-icon v-if="c.id === d.chosen" size="12" start>mdi-check</v-icon>
|
||||
{{ c.id }} · {{ c.kind }}
|
||||
</v-chip>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p v-if="d.reason" class="text-sm text-white/70 italic">„{{ d.reason }}“</p>
|
||||
|
||||
<div class="text-xs text-white/40 font-mono">
|
||||
State: {{ d.stateId }} · Flow: {{ d.flowSlug || '—' }} · Session: {{ d.sessionId.slice(0, 8) }}
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</div>
|
||||
|
||||
<p v-else class="py-12 text-center text-sm text-white/60">Keine LLM-Entscheidungen vorhanden.</p>
|
||||
|
||||
<div class="flex flex-col items-center justify-between gap-3 sm:flex-row">
|
||||
<div class="text-xs text-white/50">
|
||||
Seite {{ llmPagination.page }} von {{ llmPagination.pages }} · {{ llmPagination.total }} Einträge
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<v-btn variant="text" color="cyan" :disabled="llmPagination.page <= 1" @click="changeLlmPage(llmPagination.page - 1)">Zurück</v-btn>
|
||||
<v-btn variant="text" color="cyan" :disabled="llmPagination.page >= llmPagination.pages" @click="changeLlmPage(llmPagination.page + 1)">Weiter</v-btn>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</v-window-item>
|
||||
|
||||
<v-window-item value="bug-reports">
|
||||
<section class="space-y-6">
|
||||
<div class="flex flex-col gap-4 sm:flex-row sm:items-center sm:justify-between">
|
||||
@@ -1449,7 +1559,7 @@ useHead({ title: 'Admin • OpenSquawk' })
|
||||
const auth = useAuthStore()
|
||||
const api = useApi()
|
||||
|
||||
const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'bug-reports'>('overview')
|
||||
const activeTab = ref<'overview' | 'users' | 'invitations' | 'waitlist' | 'logs' | 'llm-routing' | 'bug-reports'>('overview')
|
||||
const refreshing = ref(false)
|
||||
|
||||
const overview = ref<OverviewData | null>(null)
|
||||
@@ -2053,6 +2163,8 @@ watch(activeTab, (tab) => {
|
||||
fetchWaitlist(true)
|
||||
} else if (tab === 'logs' && !sessionsLoaded.value) {
|
||||
fetchSessions(true)
|
||||
} else if (tab === 'llm-routing' && !llmLoaded.value) {
|
||||
fetchLlmRouting(true)
|
||||
} else if (tab === 'bug-reports' && !bugReportsLoaded.value) {
|
||||
fetchBugReports(true)
|
||||
}
|
||||
@@ -2145,6 +2257,88 @@ function changeBugReportPage(page: number) {
|
||||
watch(bugReportStatusFilter, () => fetchBugReports(true))
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// ── LLM Routing ──────────────────────────────────────────────────────────────
|
||||
interface LlmRoutingItem {
|
||||
id: string
|
||||
sessionId: string
|
||||
flowSlug?: string
|
||||
stateId: string
|
||||
transcript: string
|
||||
expectedPhrase?: string
|
||||
candidates: { id: string; label?: string; kind: 'ok' | 'bad' }[]
|
||||
chosen: string | null
|
||||
reason?: string
|
||||
status: 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'
|
||||
model: string
|
||||
timeoutMs: number
|
||||
latencyMs: number
|
||||
costUsd?: number
|
||||
createdAt: string | null
|
||||
}
|
||||
|
||||
interface LlmRoutingResponse {
|
||||
items: LlmRoutingItem[]
|
||||
counts: Record<string, number>
|
||||
pagination: { total: number; page: number; pageSize: number; pages: number }
|
||||
}
|
||||
|
||||
const llmDecisions = ref<LlmRoutingItem[]>([])
|
||||
const llmPagination = reactive({ total: 0, page: 1, pages: 1, pageSize: 20 })
|
||||
const llmLoading = ref(false)
|
||||
const llmError = ref('')
|
||||
const llmLoaded = ref(false)
|
||||
const llmStatusFilter = ref<'all' | 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'>('all')
|
||||
const llmCounts = ref<Record<string, number>>({})
|
||||
|
||||
const llmStatusItems = computed(() => [
|
||||
{ title: `Alle${llmCounts.value.all != null ? ` (${llmCounts.value.all})` : ''}`, value: 'all' },
|
||||
{ title: `Entschieden${llmCounts.value.decided != null ? ` (${llmCounts.value.decided})` : ''}`, value: 'decided' },
|
||||
{ title: `Abstain${llmCounts.value.abstain != null ? ` (${llmCounts.value.abstain})` : ''}`, value: 'abstain' },
|
||||
{ title: `Invalid${llmCounts.value.invalid != null ? ` (${llmCounts.value.invalid})` : ''}`, value: 'invalid' },
|
||||
{ title: `Timeout${llmCounts.value.timeout != null ? ` (${llmCounts.value.timeout})` : ''}`, value: 'timeout' },
|
||||
{ title: `Error${llmCounts.value.error != null ? ` (${llmCounts.value.error})` : ''}`, value: 'error' },
|
||||
])
|
||||
|
||||
function llmStatusColor(status: string) {
|
||||
switch (status) {
|
||||
case 'decided': return 'green'
|
||||
case 'abstain': return 'grey'
|
||||
case 'invalid': return 'orange'
|
||||
case 'timeout': return 'red'
|
||||
default: return 'red'
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchLlmRouting(resetPage = false) {
|
||||
if (resetPage) llmPagination.page = 1
|
||||
llmLoading.value = true
|
||||
llmError.value = ''
|
||||
try {
|
||||
const response = await api.get<LlmRoutingResponse>('/api/admin/llm-routing', {
|
||||
query: { status: llmStatusFilter.value, page: llmPagination.page },
|
||||
})
|
||||
llmDecisions.value = response.items
|
||||
Object.assign(llmPagination, response.pagination)
|
||||
const counts = { ...response.counts }
|
||||
counts.all = Object.values(response.counts).reduce((a, b) => a + b, 0)
|
||||
llmCounts.value = counts
|
||||
llmLoaded.value = true
|
||||
} catch (error) {
|
||||
llmError.value = extractErrorMessage(error, 'LLM-Entscheidungen konnten nicht geladen werden.')
|
||||
} finally {
|
||||
llmLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function changeLlmPage(page: number) {
|
||||
if (page < 1 || page > llmPagination.pages) return
|
||||
llmPagination.page = page
|
||||
fetchLlmRouting()
|
||||
}
|
||||
|
||||
watch(llmStatusFilter, () => fetchLlmRouting(true))
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
onMounted(() => {
|
||||
loadOverview(true)
|
||||
// Load open bug report count for the badge
|
||||
|
||||
64
server/api/admin/llm-routing/index.get.ts
Normal file
64
server/api/admin/llm-routing/index.get.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import { defineEventHandler, getQuery } from 'h3'
|
||||
import { requireAdmin } from '../../../utils/auth'
|
||||
import { LlmRoutingDecision } from '../../../models/LlmRoutingDecision'
|
||||
|
||||
const STATUSES = ['decided', 'abstain', 'invalid', 'timeout', 'error'] as const
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
await requireAdmin(event)
|
||||
|
||||
const query = getQuery(event)
|
||||
const page = Math.max(parseInt(String(query.page ?? '1'), 10) || 1, 1)
|
||||
const pageSize = 20
|
||||
const skip = (page - 1) * pageSize
|
||||
const status = String(query.status ?? 'all')
|
||||
|
||||
const filter: Record<string, any> = {}
|
||||
if ((STATUSES as readonly string[]).includes(status)) {
|
||||
filter.status = status
|
||||
}
|
||||
|
||||
const [docs, total, statusCounts] = await Promise.all([
|
||||
LlmRoutingDecision.find(filter)
|
||||
.sort({ createdAt: -1 })
|
||||
.skip(skip)
|
||||
.limit(pageSize)
|
||||
.lean()
|
||||
.exec(),
|
||||
LlmRoutingDecision.countDocuments(filter),
|
||||
LlmRoutingDecision.aggregate([
|
||||
{ $group: { _id: '$status', count: { $sum: 1 } } },
|
||||
]),
|
||||
])
|
||||
|
||||
const counts = Object.fromEntries(statusCounts.map((s: any) => [s._id, s.count]))
|
||||
|
||||
const items = docs.map((doc: any) => ({
|
||||
id: String(doc._id),
|
||||
sessionId: doc.sessionId,
|
||||
flowSlug: doc.flowSlug,
|
||||
stateId: doc.stateId,
|
||||
transcript: doc.transcript,
|
||||
expectedPhrase: doc.expectedPhrase,
|
||||
candidates: (doc.candidates || []).map((c: any) => ({ id: c.id, label: c.label, kind: c.kind })),
|
||||
chosen: doc.chosen ?? null,
|
||||
reason: doc.reason,
|
||||
status: doc.status,
|
||||
model: doc.model,
|
||||
timeoutMs: doc.timeoutMs,
|
||||
latencyMs: doc.latencyMs,
|
||||
costUsd: doc.costUsd,
|
||||
createdAt: doc.createdAt ? new Date(doc.createdAt).toISOString() : null,
|
||||
}))
|
||||
|
||||
return {
|
||||
items,
|
||||
counts,
|
||||
pagination: {
|
||||
total,
|
||||
page,
|
||||
pageSize,
|
||||
pages: Math.ceil(total / pageSize) || 1,
|
||||
},
|
||||
}
|
||||
})
|
||||
175
server/api/decision/route.post.ts
Normal file
175
server/api/decision/route.post.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
// server/api/decision/route.post.ts
|
||||
//
|
||||
// Internal endpoint the Python decision backend calls when regex routing fails
|
||||
// to match a pilot transmission. It asks the LLM to pick the best candidate
|
||||
// transition for the (often STT-garbled) transcript, records the cost in the
|
||||
// central usage ledger, and persists a routing-review record — including
|
||||
// timeouts — so the time budget can be tuned against real latency.
|
||||
|
||||
import { createError, readBody } from 'h3'
|
||||
import { getOpenAIClient } from '../../utils/openai'
|
||||
import { requireServiceSecret } from '../../utils/serviceAuth'
|
||||
import { recordUsage, estimateCostUsd } from '../../utils/usage'
|
||||
import {
|
||||
LlmRoutingDecision,
|
||||
type LlmRoutingCandidate,
|
||||
type LlmRoutingStatus,
|
||||
} from '../../models/LlmRoutingDecision'
|
||||
|
||||
interface RouteRequestBody {
|
||||
sessionId: string
|
||||
flowSlug?: string
|
||||
stateId: string
|
||||
transcript: string
|
||||
expectedPhrase?: string
|
||||
candidates: LlmRoutingCandidate[]
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
const ROUTER_MODEL = (process.env.ROUTER_LLM_MODEL || 'gpt-5-mini').trim()
|
||||
const DEFAULT_TIMEOUT_MS = 10_000
|
||||
|
||||
const SYSTEM_PROMPT = [
|
||||
'You are a routing classifier for an ATC radio-communication trainer.',
|
||||
'A pilot transmission was transcribed by speech-to-text and a deterministic regex layer could NOT match it to a next state.',
|
||||
'The transcript is frequently garbled by STT errors (split words, wrong numbers, homophones).',
|
||||
'Compare the transcript against the expected pilot phrase and choose the single candidate transition whose intent best matches what the pilot most likely said.',
|
||||
'Candidates are marked kind="ok" (a correct/expected radio call) or kind="bad" (an incorrect or incomplete call).',
|
||||
'Only choose a candidate id from the provided list. If none plausibly matches, choose "none".',
|
||||
'Respond with strict JSON only: {"chosen": "<candidate id or none>", "reason": "<one short sentence>"}.',
|
||||
].join(' ')
|
||||
|
||||
function buildUserPrompt(body: RouteRequestBody): string {
|
||||
const lines: string[] = []
|
||||
lines.push(`Current state: ${body.stateId}${body.flowSlug ? ` (flow ${body.flowSlug})` : ''}`)
|
||||
if (body.expectedPhrase) {
|
||||
lines.push(`Expected pilot phrase: "${body.expectedPhrase}"`)
|
||||
}
|
||||
lines.push(`Transcript (from STT): "${body.transcript}"`)
|
||||
lines.push('Candidates:')
|
||||
for (const c of body.candidates) {
|
||||
lines.push(`- id="${c.id}" kind=${c.kind}${c.label ? ` label="${c.label}"` : ''}`)
|
||||
}
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
requireServiceSecret(event)
|
||||
|
||||
const body = await readBody<RouteRequestBody>(event)
|
||||
if (!body?.transcript || !body?.stateId || !Array.isArray(body?.candidates) || body.candidates.length === 0) {
|
||||
throw createError({ statusCode: 400, statusMessage: 'transcript, stateId and candidates[] are required.' })
|
||||
}
|
||||
|
||||
const timeoutMs = Number.isFinite(body.timeoutMs) && (body.timeoutMs as number) > 0
|
||||
? (body.timeoutMs as number)
|
||||
: DEFAULT_TIMEOUT_MS
|
||||
|
||||
const candidateIds = new Set(body.candidates.map((c) => c.id))
|
||||
const client = getOpenAIClient()
|
||||
|
||||
let status: LlmRoutingStatus = 'error'
|
||||
let chosen: string | null = null
|
||||
let reason: string | undefined
|
||||
let inputTokens: number | undefined
|
||||
let outputTokens: number | undefined
|
||||
let costUsd: number | undefined
|
||||
|
||||
const started = Date.now()
|
||||
try {
|
||||
const response = await client.chat.completions.create(
|
||||
{
|
||||
model: ROUTER_MODEL,
|
||||
n: 1,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: buildUserPrompt(body) },
|
||||
],
|
||||
},
|
||||
// Per-request budget; no retries so a slow call fails fast within budget.
|
||||
{ timeout: timeoutMs, maxRetries: 0 },
|
||||
)
|
||||
|
||||
inputTokens = response.usage?.prompt_tokens
|
||||
outputTokens = response.usage?.completion_tokens
|
||||
|
||||
const raw = response.choices?.[0]?.message?.content?.trim() || ''
|
||||
let parsedChosen: string | null = null
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { chosen?: string; reason?: string }
|
||||
parsedChosen = typeof parsed.chosen === 'string' ? parsed.chosen.trim() : null
|
||||
reason = typeof parsed.reason === 'string' ? parsed.reason.trim() : undefined
|
||||
} catch {
|
||||
reason = `Unparseable model output: ${raw.slice(0, 200)}`
|
||||
}
|
||||
|
||||
if (parsedChosen && candidateIds.has(parsedChosen)) {
|
||||
chosen = parsedChosen
|
||||
status = 'decided'
|
||||
} else if (parsedChosen && parsedChosen.toLowerCase() === 'none') {
|
||||
status = 'abstain'
|
||||
} else if (parsedChosen) {
|
||||
// Model named a state outside the allowed set — never trust it.
|
||||
status = 'invalid'
|
||||
reason = reason || `Model returned out-of-set candidate "${parsedChosen}"`
|
||||
} else {
|
||||
status = 'error'
|
||||
}
|
||||
} catch (err: any) {
|
||||
const name = String(err?.name || '')
|
||||
const isTimeout = name.includes('Timeout') || err?.code === 'ETIMEDOUT' || name === 'APIConnectionTimeoutError'
|
||||
status = isTimeout ? 'timeout' : 'error'
|
||||
reason = `${name || 'LLM call failed'}: ${String(err?.message || err).slice(0, 200)}`
|
||||
}
|
||||
|
||||
const latencyMs = Date.now() - started
|
||||
|
||||
// Cost only when the call actually returned token usage (not on timeout/error).
|
||||
if (inputTokens != null || outputTokens != null) {
|
||||
costUsd = estimateCostUsd({
|
||||
kind: 'llm',
|
||||
provider: 'openai',
|
||||
model: ROUTER_MODEL,
|
||||
endpoint: '/api/decision/route',
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
})
|
||||
// Fire-and-forget into the central usage ledger (attributed by sessionId;
|
||||
// backend has no user id on the runtime session yet).
|
||||
await recordUsage({
|
||||
sessionId: body.sessionId,
|
||||
kind: 'llm',
|
||||
provider: 'openai',
|
||||
model: ROUTER_MODEL,
|
||||
endpoint: '/api/decision/route',
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
})
|
||||
}
|
||||
|
||||
// Always persist the routing-review record, including timeouts/errors.
|
||||
try {
|
||||
await LlmRoutingDecision.create({
|
||||
sessionId: body.sessionId,
|
||||
flowSlug: body.flowSlug,
|
||||
stateId: body.stateId,
|
||||
transcript: body.transcript,
|
||||
expectedPhrase: body.expectedPhrase,
|
||||
candidates: body.candidates,
|
||||
chosen,
|
||||
reason,
|
||||
status,
|
||||
model: ROUTER_MODEL,
|
||||
timeoutMs,
|
||||
latencyMs,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
costUsd,
|
||||
})
|
||||
} catch (e) {
|
||||
console.warn('[decision/route] persisting routing decision failed', e)
|
||||
}
|
||||
|
||||
return { chosen, reason, status, latencyMs, timeoutMs, model: ROUTER_MODEL }
|
||||
})
|
||||
80
server/models/LlmRoutingDecision.ts
Normal file
80
server/models/LlmRoutingDecision.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import mongoose from 'mongoose'
|
||||
|
||||
const { Schema } = mongoose
|
||||
|
||||
// Outcome of a single LLM routing attempt. `decided` = the model picked a valid
|
||||
// candidate; `abstain` = it returned "none"; `invalid` = it returned something
|
||||
// outside the candidate set; `timeout` = the request hit the time limit;
|
||||
// `error` = transport/parse failure. Timeouts and errors are persisted too so
|
||||
// the configured time limit can be tuned against real latency.
|
||||
export type LlmRoutingStatus = 'decided' | 'abstain' | 'invalid' | 'timeout' | 'error'
|
||||
|
||||
export interface LlmRoutingCandidate {
|
||||
id: string
|
||||
label?: string
|
||||
kind: 'ok' | 'bad'
|
||||
}
|
||||
|
||||
export interface LlmRoutingDecisionAttrs {
|
||||
/** Python runtime session id (no user is known backend-side yet). */
|
||||
sessionId: string
|
||||
flowSlug?: string
|
||||
stateId: string
|
||||
/** Raw STT transcript the regex layer failed to route. */
|
||||
transcript: string
|
||||
/** Rendered expected pilot phrase the model compares the transcript against. */
|
||||
expectedPhrase?: string
|
||||
candidates: LlmRoutingCandidate[]
|
||||
/** Chosen candidate id, or null when the model abstained / failed. */
|
||||
chosen: string | null
|
||||
reason?: string
|
||||
status: LlmRoutingStatus
|
||||
model: string
|
||||
/** Time budget the call was given, in ms (what to tune). */
|
||||
timeoutMs: number
|
||||
/** Wall-clock time the call actually took, in ms (incl. timeouts/errors). */
|
||||
latencyMs: number
|
||||
inputTokens?: number
|
||||
outputTokens?: number
|
||||
costUsd?: number
|
||||
createdAt: Date
|
||||
}
|
||||
|
||||
const candidateSchema = new Schema<LlmRoutingCandidate>(
|
||||
{
|
||||
id: { type: String, required: true },
|
||||
label: { type: String },
|
||||
kind: { type: String, enum: ['ok', 'bad'], required: true },
|
||||
},
|
||||
{ _id: false },
|
||||
)
|
||||
|
||||
const llmRoutingDecisionSchema = new mongoose.Schema<LlmRoutingDecisionAttrs>({
|
||||
sessionId: { type: String, required: true, index: true },
|
||||
flowSlug: { type: String },
|
||||
stateId: { type: String, required: true },
|
||||
transcript: { type: String, required: true },
|
||||
expectedPhrase: { type: String },
|
||||
candidates: { type: [candidateSchema], default: [] },
|
||||
chosen: { type: String, default: null },
|
||||
reason: { type: String },
|
||||
status: {
|
||||
type: String,
|
||||
enum: ['decided', 'abstain', 'invalid', 'timeout', 'error'],
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
model: { type: String, required: true },
|
||||
timeoutMs: { type: Number, required: true },
|
||||
latencyMs: { type: Number, required: true },
|
||||
inputTokens: { type: Number },
|
||||
outputTokens: { type: Number },
|
||||
costUsd: { type: Number },
|
||||
createdAt: { type: Date, default: () => new Date(), index: true },
|
||||
})
|
||||
|
||||
llmRoutingDecisionSchema.index({ status: 1, createdAt: -1 })
|
||||
|
||||
export const LlmRoutingDecision =
|
||||
(mongoose.models.LlmRoutingDecision as mongoose.Model<LlmRoutingDecisionAttrs> | undefined) ||
|
||||
mongoose.model<LlmRoutingDecisionAttrs>('LlmRoutingDecision', llmRoutingDecisionSchema)
|
||||
32
server/utils/serviceAuth.ts
Normal file
32
server/utils/serviceAuth.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import type { H3Event } from 'h3'
|
||||
import { createError, getHeader } from 'h3'
|
||||
|
||||
let warnedMissingSecret = false
|
||||
|
||||
/**
|
||||
* Guards internal server-to-server endpoints (e.g. the Python decision backend
|
||||
* calling back into Nuxt). The secret is read from SERVICE_SECRET and must be
|
||||
* supplied via the `x-service-secret` header.
|
||||
*
|
||||
* Fail closed: if no secret is configured the endpoint refuses to run (503)
|
||||
* rather than being publicly callable.
|
||||
*/
|
||||
export function requireServiceSecret(event: H3Event) {
|
||||
const secret = (process.env.SERVICE_SECRET || '').trim()
|
||||
|
||||
if (!secret) {
|
||||
if (!warnedMissingSecret) {
|
||||
console.error(
|
||||
'[service] SERVICE_SECRET is not set — internal endpoints are disabled (503). ' +
|
||||
'Set SERVICE_SECRET and pass it via the x-service-secret header.',
|
||||
)
|
||||
warnedMissingSecret = true
|
||||
}
|
||||
throw createError({ statusCode: 503, statusMessage: 'Service endpoint is not configured.' })
|
||||
}
|
||||
|
||||
const provided = (getHeader(event, 'x-service-secret') || '').trim()
|
||||
if (provided !== secret) {
|
||||
throw createError({ statusCode: 401, statusMessage: 'Invalid service secret.' })
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user