From 79dfc60be1e00982e2c8bc6f2098309d227d6d69 Mon Sep 17 00:00:00 2001 From: Remi <73385395+itsrubberduck@users.noreply.github.com> Date: Sat, 20 Sep 2025 12:42:08 +0200 Subject: [PATCH] Enhance admin transmissions with detailed LLM and TTS context --- app/pages/admin/index.vue | 373 ++++++++++++++++++++++++++++++++++++- server/api/atc/ptt.post.ts | 80 ++++++++ server/api/atc/say.post.ts | 13 +- 3 files changed, 462 insertions(+), 4 deletions(-) diff --git a/app/pages/admin/index.vue b/app/pages/admin/index.vue index 76188ff..2ccbe6f 100644 --- a/app/pages/admin/index.vue +++ b/app/pages/admin/index.vue @@ -729,9 +729,9 @@
-
+

LLM Decision Summary

@@ -752,6 +752,159 @@
+ +
+

State context snapshot

+
+
+
+

+ {{ entry.metadata.context.stateId || 'Unknown state' }} +

+

+ {{ entry.metadata.context.state?.name }} +

+
+
+ + Role: {{ entry.metadata.context.state?.role }} + + + Phase: {{ entry.metadata.context.state?.phase }} + +
+
+
+

State Say Template

+

{{ entry.metadata.context.state?.say_tpl }}

+
+
+
+

Next transitions

+
    +
  • + {{ describeTransition(transition) }} +
    • +
+

—

+
+
+

OK transitions

+
    +
  • + {{ describeTransition(transition) }} +
    • +
+

—

+
+
+

Bad transitions

+
    +
  • + {{ describeTransition(transition) }} +
    • +
+

—

+
+
+
+

Candidates

+
+
+ {{ candidate.id || 'unknown' }} + + Selected + +
+

+ {{ candidate.state?.name }} +

+
+ + Role: {{ candidate.state?.role }} + + + Requires ATC reply + +
+

+ {{ candidate.state?.say_tpl }} +

+
+
+
+
+

Variables snapshot

+ 
{{ formatJson(entry.metadata.context.variables) }}
+
+
+

Flags snapshot

+ 
{{ formatJson(entry.metadata.context.flags) }}
+
+
+
+

OpenAI Decision Calls

+ +
+

Say endpoint invocation

+
+
+

Voice

+

{{ entry.metadata?.voice || '—' }}

+
+
+

Signal level

+

{{ entry.metadata?.level ?? '—' }}

+
+
+

Speech speed

+

{{ entry.metadata?.speed ?? '—' }}

+
+
+

Radio quality

+

{{ entry.metadata?.radioQuality || '—' }}

+
+
+
+
+

TTS provider

+

{{ entry.metadata?.tts?.provider || '—' }}

+
+
+

Model

+

{{ entry.metadata?.tts?.model || '—' }}

+
+
+

Format

+

+ {{ entry.metadata?.tts?.format || '—' }} + + ({{ entry.metadata?.tts?.extension }}) + +

+
+
+
+

Tag

+

{{ entry.metadata?.tag }}

+
+

Metadata

{{ formatJson(entry.metadata) }}
@@ -980,6 +1189,78 @@ interface TransmissionUser { role: string } +interface DecisionTraceCall { + stage: 'readback-check' | 'decision' + request: Record + response?: any + rawResponseText?: string + error?: string +} + +interface DecisionTraceMetadata { + calls?: DecisionTraceCall[] + fallback?: { used?: boolean; reason?: string; selected?: string } +} + +interface CandidateSnapshot { + id?: string + state?: Record +} + +interface TransmissionContextSnapshot { + stateId?: string + state?: Record + candidates?: CandidateSnapshot[] + selectedCandidate?: CandidateSnapshot + variables?: Record + flags?: Record +} + +interface LlmUsageMetadata { + autoDecide?: boolean + openaiUsed?: boolean + callCount?: number + fallbackUsed?: boolean + strategy?: 'manual' | 'openai' | 'heuristic' | 'fallback' + reason?: string +} + +interface TransmissionMetadata { + moduleId?: string + lessonId?: string + autoDecide?: boolean + decision?: { + next_state?: string + controller_say_tpl?: string + off_schema?: boolean + radio_check?: boolean + } + decisionTrace?: DecisionTraceMetadata + context?: TransmissionContextSnapshot + llm?: LlmUsageMetadata + tts?: { + provider?: string + model?: string + format?: string + extension?: string + } + voice?: string + level?: number + speed?: number + tag?: string | null + radioQuality?: string + [key: string]: any +} + +interface LlmUsageSummary { + method: string + openaiUsed: boolean + callCount: number + fallbackUsed: boolean + autoDecide?: boolean + reason?: string +} + interface TransmissionEntry { id: string role: string @@ -988,7 +1269,7 @@ interface TransmissionEntry { text: string normalized?: string createdAt: string - metadata?: Record + metadata?: TransmissionMetadata user?: TransmissionUser } @@ -1203,6 +1484,92 @@ function formatJson(value?: any) { } } +function transitionList(value: any) { + return Array.isArray(value) ? value : [] +} + +function describeTransition(transition: any) { + if (!transition) return '—' + if (typeof transition === 'string') return transition + if (typeof transition !== 'object') return String(transition) + + const destination = transition.to || transition.id || '—' + const details: string[] = [] + + if (transition.when) details.push(`when: ${transition.when}`) + if (transition.action) details.push(`action: ${transition.action}`) + if (transition.intent) details.push(`intent: ${transition.intent}`) + if (transition.auto) details.push(`auto: ${transition.auto}`) + if (transition.say_tpl || transition.controller_say_tpl) details.push('say') + if (transition.note) details.push(`note: ${transition.note}`) + if (transition.condition) details.push(`cond: ${transition.condition}`) + + return details.length ? `${destination} (${details.join(', ')})` : destination +} + +function buildLlmUsage(entry: TransmissionEntry): LlmUsageSummary | null { + const metadata = entry.metadata + if (!metadata) return null + + const hasDecisionData = + metadata.autoDecide !== undefined || + Boolean(metadata.llm) || + Boolean(metadata.decisionTrace?.calls?.length) || + Boolean(metadata.decisionTrace?.fallback?.used) + + if (!hasDecisionData) return null + + const autoDecide = metadata.llm?.autoDecide ?? metadata.autoDecide + const callCount = metadata.llm?.callCount ?? metadata.decisionTrace?.calls?.length ?? 0 + const fallbackUsed = metadata.llm?.fallbackUsed ?? Boolean(metadata.decisionTrace?.fallback?.used) + const openaiUsed = metadata.llm?.openaiUsed ?? callCount > 0 + + const strategy = + metadata.llm?.strategy || + (!autoDecide + ? 'manual' + : openaiUsed + ? 'openai' + : fallbackUsed + ? 'fallback' + : 'heuristic') + + let method: string + switch (strategy) { + case 'openai': + method = `OpenAI decision (${callCount} ${callCount === 1 ? 'call' : 'calls'})` + break + case 'fallback': + method = 'Fallback decision after OpenAI error' + break + case 'manual': + method = 'Manual routing (auto decision disabled)' + break + default: + method = 'Heuristic decision (no OpenAI call)' + break + } + + const reason = + metadata.llm?.reason || + (strategy === 'openai' + ? `Decision derived from OpenAI with ${callCount} ${callCount === 1 ? 'call' : 'calls'}.` + : strategy === 'fallback' + ? metadata.decisionTrace?.fallback?.reason || 'Fallback executed because OpenAI response could not be used.' + : strategy === 'manual' + ? 'Automatic decision was disabled for this transmission.' + : 'Rules and heuristics resolved the decision without contacting OpenAI.') + + return { + method, + openaiUsed, + callCount, + fallbackUsed, + autoDecide, + reason, + } +} + function isExpired(expiresAt?: string) { if (!expiresAt) return false const date = new Date(expiresAt) diff --git a/server/api/atc/ptt.post.ts b/server/api/atc/ptt.post.ts index ed6c58f..d7f1a88 100644 --- a/server/api/atc/ptt.post.ts +++ b/server/api/atc/ptt.post.ts @@ -88,6 +88,19 @@ async function convertToWav(inputPath: string, outputPath: string) { ]); } +function safeClone(value: T): T | undefined { + if (value === undefined) { + return undefined; + } + + try { + return JSON.parse(JSON.stringify(value)); + } catch (err) { + console.warn("Failed to clone value for transmission metadata", err); + return undefined; + } +} + export default defineEventHandler(async (event) => { const body = await readBody(event); @@ -161,6 +174,64 @@ export default defineEventHandler(async (event) => { try { const user = await getUserFromEvent(event) + + const llmCallCount = decisionResult?.trace?.calls?.length || 0; + const fallbackUsed = Boolean(decisionResult?.trace?.fallback?.used); + + let llmStrategy: 'manual' | 'openai' | 'heuristic' | 'fallback' = 'manual'; + if (shouldAutoDecide) { + if (llmCallCount > 0) { + llmStrategy = 'openai'; + } else if (fallbackUsed) { + llmStrategy = 'fallback'; + } else { + llmStrategy = 'heuristic'; + } + } + + const llmUsage = { + autoDecide: shouldAutoDecide, + openaiUsed: llmStrategy === 'openai', + callCount: llmCallCount, + fallbackUsed, + strategy: llmStrategy, + reason: + llmStrategy === 'manual' + ? 'Automatic decision disabled in request.' + : llmStrategy === 'openai' + ? `Decision derived from OpenAI with ${llmCallCount} call(s).` + : llmStrategy === 'fallback' + ? (decisionResult?.trace?.fallback?.reason || 'Fallback triggered after OpenAI failure.') + : 'Decision resolved locally without calling OpenAI.' + }; + + const contextState = safeClone(body.context.state); + if (contextState && typeof contextState === 'object' && contextState !== null) { + const stateRecord = contextState as Record; + if (!('id' in stateRecord)) { + stateRecord.id = body.context.state_id; + } + } + + const contextCandidates = Array.isArray(body.context.candidates) + ? body.context.candidates.map(candidate => { + const candidateState = safeClone(candidate.state); + if (candidateState && typeof candidateState === 'object' && candidateState !== null) { + const candidateRecord = candidateState as Record; + if (!('id' in candidateRecord)) { + candidateRecord.id = candidate.id; + } + } + + return { + id: candidate.id, + state: candidateState + }; + }) + : undefined; + + const selectedCandidate = contextCandidates?.find(c => c.id === decision?.next_state); + await TransmissionLog.create({ user: user?._id, role: "pilot", @@ -173,6 +244,15 @@ export default defineEventHandler(async (event) => { decision, decisionTrace: decisionResult?.trace, autoDecide: shouldAutoDecide, + llm: llmUsage, + context: { + stateId: body.context.state_id, + state: contextState, + candidates: contextCandidates, + selectedCandidate, + variables: safeClone(body.context.variables), + flags: safeClone(body.context.flags) + } }, }) } catch (logError) { diff --git a/server/api/atc/say.post.ts b/server/api/atc/say.post.ts index 6c8a3d6..aa26450 100644 --- a/server/api/atc/say.post.ts +++ b/server/api/atc/say.post.ts @@ -159,6 +159,7 @@ export default defineEventHandler(async (event) => { let audioBuffer: Buffer; let modelUsed: string; let actualMime = mime; + let ttsProvider: 'openai' | 'speaches' | 'piper' = 'openai'; if (useSpeaches) { // Speaches (prefer compact: MP3, otherwise FLAC/WAV/PCM) @@ -171,12 +172,14 @@ export default defineEventHandler(async (event) => { modelUsed = model; // Server returns the correct format according to response_format actualMime = fmtToMime(fmt); + ttsProvider = 'speaches'; } else if (usePiper) { // Local Piper audioBuffer = await piperTTS(normalized, voice, runtimeConfig.piperPort); modelUsed = "piper-local"; // Piper returns WAV actualMime = "audio/wav"; + ttsProvider = 'piper'; } else { // OpenAI (fallback) const tts = await normalize.audio.speech.create({ @@ -189,6 +192,7 @@ export default defineEventHandler(async (event) => { audioBuffer = Buffer.from(await tts.arrayBuffer()); modelUsed = TTS_MODEL; actualMime = "audio/wav"; + ttsProvider = 'openai'; } // Optional persistence @@ -208,7 +212,8 @@ export default defineEventHandler(async (event) => { lessonId: body?.lessonId || null, files: { audio: fileOut }, model: modelUsed, - format: actualMime + format: actualMime, + ttsProvider }; // await writeFile(fileJson, JSON.stringify(meta, null, 2), "utf-8"); @@ -229,6 +234,12 @@ export default defineEventHandler(async (event) => { lessonId: body?.lessonId || null, tag: body?.tag || null, radioQuality: radioQuality.description, + tts: { + provider: ttsProvider, + model: modelUsed, + format: actualMime, + extension: ext + } } }) } catch (logError) {