diff --git a/app/pages/pm.vue b/app/pages/pm.vue index 216e9de..ab985e8 100644 --- a/app/pages/pm.vue +++ b/app/pages/pm.vue @@ -356,7 +356,7 @@

Quick Actions

-
+
Radio Check - - Reset Log -
@@ -403,25 +394,64 @@ - - - + + +
-

Debug: Next States

- LLM Targets +

Debug: Flow Insights

+ LLM
-
- - {{ nid }} - + +
+

Aktueller Knoten

+

{{ debugState?.id || '—' }}

+

+ {{ debugState ? `${debugState.role} • ${debugState.phase}` : 'N/A' }} + ({{ debugState.frequencyName }}) +

+

+ Auto: {{ debugState.sayPlain }} +

+

+ Funk: {{ debugState.sayNormalized }} +

+
+ +
+

Nächste Entscheidungen

+
+
+
+
+

{{ state.id }}

+

+ {{ state.role || '—' }} • {{ state.phase || '—' }} + ({{ state.frequencyName }}) +

+
+ + Jump + +
+

+ ATC: {{ state.sayPlain }} +

+

+ Funk: {{ state.sayNormalized }} +

+
+
+

Keine weiteren Entscheidungen verfügbar.

@@ -431,16 +461,7 @@

Communication Log

-
- - {{ log.length }} -
+ {{ log.length }}
@@ -509,7 +530,7 @@ />
-
+
+ { return { color: 'error', text: 'WEAK' } }) +type PreparedSpeech = { + template: string + plain: string + normalized: string +} + +type SpeechOptions = { + voice?: string + tag?: string + updateLastTransmission?: boolean + lastTransmissionLabel?: string + delayMs?: number + useNormalizedForTTS?: boolean +} + +const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +let audioContext: AudioContext | null = null +let speechQueue: Promise = Promise.resolve() + +const enqueueSpeech = (task: () => Promise) => { + speechQueue = speechQueue + .then(() => task()) + .catch(err => { + console.error('Speech queue error:', err) + }) + return speechQueue +} + +const prepareSpeech = (tpl: string): PreparedSpeech => { + const plain = renderATCMessage(tpl) + const normalized = normalizeATCText(tpl, { ...vars.value, ...flags.value }) + return { template: tpl, plain, normalized } +} + +const debugState = computed(() => { + if (!currentState.value) return null + const sayTpl = currentState.value.say_tpl + return { + id: currentState.value.id, + phase: currentState.value.phase, + role: currentState.value.role, + frequencyName: currentState.value.frequencyName, + sayPlain: sayTpl ? renderATCMessage(sayTpl) : '', + sayNormalized: sayTpl ? normalizeATCText(sayTpl, { ...vars.value, ...flags.value }) : '' + } +}) + +const debugNextStates = computed(() => { + return nextCandidates.value.map(id => { + const state = getStateDetails(id) + if (!state) { + return { id, role: '', phase: '', frequencyName: undefined, sayPlain: '', sayNormalized: '' } + } + const sayPlain = state.say_tpl ? renderATCMessage(state.say_tpl) : '' + const sayNormalized = state.say_tpl ? normalizeATCText(state.say_tpl, { ...vars.value, ...flags.value }) : '' + return { + id, + role: state.role, + phase: state.phase, + frequencyName: state.frequencyName, + sayPlain, + sayNormalized + } + }) +}) + // Methods const normalizeExpectedText = (text: string): string => { if (!flightContext.value) return text return normalizeATCText(text, { ...vars.value, ...flags.value }) } +const ensureAudioContext = async (): Promise => { + if (typeof window === 'undefined') return null + + if (!audioContext) { + const Ctx = window.AudioContext || (window as any).webkitAudioContext + if (!Ctx) return null + audioContext = new Ctx() + } + + if (audioContext.state === 'suspended') { + try { + await audioContext.resume() + } catch (err) { + console.warn('AudioContext resume failed', err) + } + } + + return audioContext +} + +const playAudioWithEffects = async (base64: string) => { + if (typeof window === 'undefined') return + + if (!radioEffectsEnabled.value) { + await new Promise((resolve) => { + const audio = new Audio(`data:audio/wav;base64,${base64}`) + audio.onended = () => resolve() + audio.onerror = () => resolve() + audio.play().catch(() => resolve()) + }) + return + } + + try { + const ctx = await ensureAudioContext() + if (!ctx) throw new Error('AudioContext unavailable') + + const binary = Uint8Array.from(atob(base64), c => c.charCodeAt(0)) + const arrayBuffer = binary.buffer.slice(binary.byteOffset, binary.byteOffset + binary.byteLength) + const buffer = await ctx.decodeAudioData(arrayBuffer) + + await new Promise((resolve) => { + const source = ctx.createBufferSource() + source.buffer = buffer + + const highpass = ctx.createBiquadFilter() + highpass.type = 'highpass' + highpass.frequency.value = 320 + + const lowpass = ctx.createBiquadFilter() + lowpass.type = 'lowpass' + lowpass.frequency.value = 3100 + + const compressor = ctx.createDynamicsCompressor() + compressor.threshold.value = -28 + compressor.knee.value = 30 + compressor.ratio.value = 12 + compressor.attack.value = 0.003 + compressor.release.value = 0.25 + + const gainNode = ctx.createGain() + gainNode.gain.value = 0.9 + + source.connect(highpass) + highpass.connect(lowpass) + lowpass.connect(compressor) + compressor.connect(gainNode) + gainNode.connect(ctx.destination) + + let noiseSource: AudioBufferSourceNode | null = null + + if (buffer.duration > 0) { + const length = Math.ceil(buffer.duration * ctx.sampleRate) + const noiseBuffer = ctx.createBuffer(1, length, ctx.sampleRate) + const channel = noiseBuffer.getChannelData(0) + const strength = Math.max(1, Math.min(5, signalStrength.value)) + const intensity = (6 - strength) / 6 + const amplitude = 0.015 + intensity * 0.045 + for (let i = 0; i < channel.length; i++) { + channel[i] = (Math.random() * 2 - 1) * amplitude + } + noiseSource = ctx.createBufferSource() + noiseSource.buffer = noiseBuffer + const bandPass = ctx.createBiquadFilter() + bandPass.type = 'bandpass' + bandPass.frequency.value = 1800 + bandPass.Q.value = 1.2 + const noiseGain = ctx.createGain() + noiseGain.gain.value = amplitude * 0.6 + noiseSource.connect(bandPass) + bandPass.connect(noiseGain) + noiseGain.connect(ctx.destination) + noiseSource.start(0) + } + + source.onended = () => { + if (noiseSource) { + try { + noiseSource.stop() + } catch (err) { + // ignore + } + } + resolve() + } + + source.start(0) + }) + } catch (err) { + console.error('Failed to apply radio effect', err) + await new Promise((resolve) => { + const audio = new Audio(`data:audio/wav;base64,${base64}`) + audio.onended = () => resolve() + audio.onerror = () => resolve() + audio.play().catch(() => resolve()) + }) + } +} + +const speakPrepared = async (prepared: PreparedSpeech, options: SpeechOptions = {}) => { + try { + const response = await $fetch('/api/atc/say', { + method: 'POST', + body: { + text: options.useNormalizedForTTS === false ? prepared.plain : prepared.normalized, + level: signalStrength.value, + voice: options.voice || 'alloy', + speed: 0.95, + moduleId: 'pilot-monitoring', + lessonId: currentState.value?.id || 'general', + tag: options.tag || 'controller-reply' + } + }) + + if (response.success && response.audio) { + if (options.updateLastTransmission !== false) { + lastTransmission.value = options.lastTransmissionLabel || `ATC: ${prepared.plain}` + } + await playAudioWithEffects(response.audio.base64) + } + } catch (err) { + console.error('TTS failed:', err) + } +} + +const speakWithRadioEffects = (tpl: string, options: SpeechOptions = {}) => { + const prepared = prepareSpeech(tpl) + const delay = options.delayMs ?? 0 + enqueueSpeech(async () => { + if (delay > 0) { + await wait(delay) + } + await speakPrepared(prepared, options) + }) +} + +const scheduleControllerSpeech = (tpl: string) => { + const plain = renderATCMessage(tpl) + speakWithRadioEffects(tpl, { + delayMs: 800 + Math.random() * 2000, + tag: 'controller-reply', + updateLastTransmission: true, + useNormalizedForTTS: true, + lastTransmissionLabel: `ATC: ${plain}` + }) +} + +const speakPilotReadback = (text: string) => { + speakWithRadioEffects(text, { + delayMs: 400, + voice: 'verse', + tag: 'pilot-readback', + updateLastTransmission: false, + useNormalizedForTTS: true + }) +} + +const handlePilotTransmission = async (message: string, source: 'text' | 'ptt' = 'text') => { + const transcript = message.trim() + if (!transcript) return + + const prefix = source === 'ptt' ? 'Pilot (PTT)' : 'Pilot' + lastTransmission.value = `${prefix}: ${transcript}` + + const quickResponse = processPilotTransmission(transcript) + + if (readbackEnabled.value) { + speakPilotReadback(transcript) + } + + if (quickResponse) { + scheduleControllerSpeech(quickResponse) + return + } + + const ctx = buildLLMContext(transcript) + + try { + const decision = await $fetch('/api/llm/decide', { + method: 'POST', + body: ctx + }) + + applyLLMDecision(decision) + + if (decision.controller_say_tpl && !decision.radio_check) { + scheduleControllerSpeech(decision.controller_say_tpl) + } + } catch (e) { + console.error('LLM decision failed', e) + lastTransmission.value = `${prefix}: ${transcript} (LLM failed)` + } +} + // VATSIM Integration const loadFlightPlans = async () => { if (!vatsimId.value) return @@ -654,7 +966,6 @@ const backToSetup = () => { currentScreen.value = 'login' selectedPlan.value = null lastTransmission.value = '' - resetCommunications() } // Audio/PTT Functions @@ -724,17 +1035,15 @@ const stopRecording = () => { const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => { try { - // Convert blob to base64 const arrayBuffer = await audioBlob.arrayBuffer() const base64Audio = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer))) if (isIntercom) { - // Handle intercom separately - simple transcription only const result = await $fetch('/api/atc/ptt', { method: 'POST', body: { audio: base64Audio, - context: { // Minimal context for intercom + context: { state_id: currentState.value?.id || 'INTERCOM', state: {}, candidates: [], @@ -743,7 +1052,8 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => { }, moduleId: 'pilot-monitoring-intercom', lessonId: 'intercom', - format: 'webm' + format: 'webm', + autoDecide: false } }) @@ -751,36 +1061,31 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => { lastTransmission.value = `INTERCOM: ${result.transcription}` const transcription = result.transcription.toLowerCase() if (transcription.includes('checklist') || transcription.includes('check list')) { - setTimeout(() => speakWithRadioEffects('Checklist functionality available in advanced mode'), 1000) + speakWithRadioEffects('Checklist functionality available in advanced mode', { + delayMs: 600, + updateLastTransmission: false, + tag: 'system-info', + useNormalizedForTTS: true + }) } } } else { - // Handle ATC communication - full PTT + Decision in one call - const ctx = buildLLMContext('') // Build context first + const ctx = buildLLMContext('') const result = await $fetch('/api/atc/ptt', { method: 'POST', body: { audio: base64Audio, - context: ctx, // Full LLM context + context: ctx, moduleId: 'pilot-monitoring', lessonId: currentState.value?.id || 'general', - format: 'webm' + format: 'webm', + autoDecide: false } }) if (result.success) { - lastTransmission.value = result.transcription - - // Apply the decision directly from PTT response - applyLLMDecision(result.decision) - - // If ATC should respond, speak it - if (result.decision.controller_say_tpl && !result.decision.radio_check) { - setTimeout(async () => { - await speakWithRadioEffects(result.decision.controller_say_tpl!) - }, 1000 + Math.random() * 2000) - } + await handlePilotTransmission(result.transcription, 'ptt') } } } catch (err) { @@ -789,94 +1094,12 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean) => { } } -const processPilotAndLLM = async (transcript: string) => { - // Process pilot transmission (logs it, handles basic routing like radio checks/emergencies) - const quickResponse = processPilotTransmission(transcript) - - if (quickResponse) { - // Radio check or emergency was handled directly - return - } - - // Build context for LLM - const ctx = buildLLMContext(transcript) - - try { - const decision = await decideNextStateLLM(ctx) - applyLLMDecision(decision) - - // If ATC should respond, speak it - if (decision.controller_say_tpl && !decision.radio_check) { - setTimeout(async () => { - await speakWithRadioEffects(decision.controller_say_tpl!) - }, 1000 + Math.random() * 2000) - } - } catch (e) { - console.error('LLM decision failed', e) - lastTransmission.value += ' (LLM failed - logged only)' - } -} - const sendPilotText = async () => { const text = pilotInput.value.trim() if (!text) return pilotInput.value = '' - lastTransmission.value = text - - // Process text input directly through decision engine - const quickResponse = processPilotTransmission(text) - - if (quickResponse) { - // Radio check or emergency was handled directly - return - } - - // Build context and get LLM decision - const ctx = buildLLMContext(text) - - try { - const decision = await $fetch('/api/llm/decide', { - method: 'POST', - body: ctx - }) - - applyLLMDecision(decision) - - // If ATC should respond, speak it - if (decision.controller_say_tpl && !decision.radio_check) { - setTimeout(async () => { - await speakWithRadioEffects(decision.controller_say_tpl!) - }, 1000 + Math.random() * 2000) - } - } catch (e) { - console.error('LLM decision failed', e) - lastTransmission.value += ' (LLM failed - logged only)' - } -} - -const speakWithRadioEffects = async (text: string) => { - try { - const response = await $fetch('/api/atc/say', { - method: 'POST', - body: { - text, - level: signalStrength.value, - voice: 'alloy', - speed: 0.95, - moduleId: 'pilot-monitoring', - lessonId: currentState.value?.id || 'general' - } - }) - - if (response.success && response.audio) { - // Play the audio - const audio = new Audio(`data:audio/wav;base64,${response.audio.base64}`) - audio.play().catch(console.error) - } - } catch (err) { - console.error('TTS failed:', err) - } + await handlePilotTransmission(text, 'text') } const performRadioCheck = async () => { @@ -885,14 +1108,12 @@ const performRadioCheck = async () => { radioCheckLoading.value = true const message = `${frequencies.value.active}, ${flightContext.value.callsign}, radio check` - lastTransmission.value = message try { - // Process as pilot transmission - await processPilotAndLLM(message) - radioCheckLoading.value = false + await handlePilotTransmission(message, 'text') } catch (err) { console.error('Radio check failed:', err) + } finally { radioCheckLoading.value = false } } @@ -909,16 +1130,6 @@ const swapFrequencies = () => { }, 500) } -const resetCommunications = () => { - log.value.splice(0) - lastTransmission.value = '' - lastEvaluation.value = null -} - -const clearCommunicationLog = () => { - log.value.splice(0) -} - const formatTime = (date: Date): string => { return date.toLocaleTimeString('de-DE', { hour12: false, diff --git a/server/api/atc/ptt.post.ts b/server/api/atc/ptt.post.ts index 494f095..f340d8e 100644 --- a/server/api/atc/ptt.post.ts +++ b/server/api/atc/ptt.post.ts @@ -20,12 +20,13 @@ interface PTTRequest { moduleId: string; lessonId: string; format?: 'wav' | 'mp3' | 'ogg' | 'webm'; + autoDecide?: boolean; } interface PTTResponse { success: boolean; transcription: string; - decision: { + decision?: { next_state: string; controller_say_tpl?: string; off_schema?: boolean; @@ -99,13 +100,19 @@ export default defineEventHandler(async (event) => { }); } - // 4. Direkt LLM Decision aufrufen mit transkribiertem Text - const decisionInput = { - ...body.context, - pilot_utterance: transcribedText - }; + const shouldAutoDecide = body.autoDecide !== false; - const decision = await routeDecision(decisionInput); + let decision: PTTResponse['decision']; + + if (shouldAutoDecide) { + // 4. Direkt LLM Decision aufrufen mit transkribiertem Text + const decisionInput = { + ...body.context, + pilot_utterance: transcribedText + }; + + decision = await routeDecision(decisionInput); + } // 5. Cleanup await rm(tmpAudioInput).catch(() => {}); @@ -115,10 +122,13 @@ export default defineEventHandler(async (event) => { const result: PTTResponse = { success: true, - transcription: transcribedText, - decision + transcription: transcribedText }; + if (decision) { + result.decision = decision; + } + return result; } catch (error: any) { diff --git a/shared/utils/communicationsEngine.ts b/shared/utils/communicationsEngine.ts index 1e3be53..e4aeb0d 100644 --- a/shared/utils/communicationsEngine.ts +++ b/shared/utils/communicationsEngine.ts @@ -472,6 +472,10 @@ export default function useCommunicationsEngine() { communicationLog.value.push(entry) } + function renderATCMessage(tpl: string) { + return renderTpl(tpl, exposeCtx()) + } + function exposeCtx() { return { variables: variables.value, flags: flags.value } } @@ -490,6 +494,12 @@ export default function useCommunicationsEngine() { return flags.value.current_unit } + function getStateDetails(stateId: string) { + const s = states.value[stateId] + if (!s) return null + return { ...s, id: stateId } + } + function genStand() { const arr = ['A12','B15','C23','D8','E41','F18','G7','H33'] return arr[Math.floor(Math.random() * arr.length)] @@ -568,6 +578,8 @@ export default function useCommunicationsEngine() { resumePriorFlow, // Utilities - normalizeATCText + normalizeATCText, + renderATCMessage, + getStateDetails } }