From ba5edcbd025b63fd3271b1ba58edd69cb4d96129 Mon Sep 17 00:00:00 2001 From: itsrubberduck Date: Tue, 26 May 2026 10:58:38 +0200 Subject: [PATCH] Add STT readback input to classroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detlef's original "Rückmeldungen sollten gesprochen werden können" request was about speaking *into* the readback rather than typing every field — this wires that up using the existing /api/atc/ptt Whisper endpoint. - Mic button in the readback control row records via MediaRecorder, posts base64 audio to /api/atc/ptt, then maps the transcription onto the lesson's input fields via normalized substring matching against each field's expected value plus alternatives. - Callsign-flagged fields additionally allow a sliding-window Levenshtein match because Whisper routinely garbles airline names. - The transcription is shown back to the user so they can verify what was heard before pressing Check. - The button is hidden when the Speaches speech server is configured but unreachable — a small "Speech server unavailable" hint takes its place. Browsers without MediaRecorder/getUserMedia never see it. - Recording state animates the button; transcription state shows a spinner. Stops cleanly on Reset and on component unmount. Co-Authored-By: Claude Opus 4.7 --- app/pages/classroom.vue | 287 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) diff --git a/app/pages/classroom.vue b/app/pages/classroom.vue index dad4463..235e304 100644 --- a/app/pages/classroom.vue +++ b/app/pages/classroom.vue @@ -1118,6 +1118,37 @@ mdi-volume-high Speak answer + + + mdi-microphone-off + Speech server unavailable + + +
+
+ mdi-alert + {{ sttError }} +
+
+ Heard: + {{ sttLastTranscription }} +
{{ result.score }}%
@@ -2790,6 +2821,23 @@ const toast = ref({show: false, text: ''}) const showSettings = ref(false) const showSpeechServerWarning = ref(false) const showOnlineTtsSuggestion = ref(false) + +// STT (Speech-to-Text) for the readback — pilot speaks the readback into a mic, +// transcription is mapped onto the input fields. Driven by /api/atc/ptt. +const sttSupported = isClient && typeof window !== 'undefined' + && typeof navigator !== 'undefined' + && Boolean(navigator.mediaDevices?.getUserMedia) + && typeof window.MediaRecorder !== 'undefined' +const sttServerAvailable = ref(true) +const sttRecording = ref(false) +const sttTranscribing = ref(false) +const sttError = ref('') +const sttLastTranscription = ref('') +const sttMediaRecorder = ref(null) +const sttChunks = ref([]) +const sttStream = ref(null) +const sttFeatureVisible = computed(() => sttSupported && sttServerAvailable.value) + const api = useApi() const isClient = typeof window !== 'undefined' const auth = useAuthStore() @@ -2861,8 +2909,12 @@ async function checkSpeechServerAvailability() { const hasOnlineTts = Boolean(health.configured && health.reachable) showSpeechServerWarning.value = Boolean(health.configured && !health.reachable) showOnlineTtsSuggestion.value = Boolean(hasOnlineTts && cfg.value.tts) + // STT: if a Speaches server is configured but down, hide the feature. + // If no Speaches is configured we assume OpenAI Whisper cloud is reachable. + sttServerAvailable.value = !health.configured || health.reachable } catch (error) { console.error('Failed to check speech server availability', error) + sttServerAvailable.value = false } } @@ -3411,6 +3463,171 @@ async function speakCorrectReadback() { await say(text) } +// --------------------------------------------------------------------------- +// STT — pilot speaks the readback into the mic, transcription is mapped onto +// the input fields. Callsigns are notoriously misrecognized by Whisper, so we +// run fuzzy substring matching with the existing alternative-list for each +// field; for callsign-flagged fields we additionally allow a sliding-window +// Levenshtein match against the transcription. +// --------------------------------------------------------------------------- + +function fuzzyContains(haystack: string, needle: string): boolean { + if (!needle || !haystack) return false + if (haystack.includes(needle)) return true + const tolerance = allowedDistance(needle.length) + 1 + const minLen = Math.max(3, needle.length - 2) + const maxLen = needle.length + 3 + for (let start = 0; start <= haystack.length - minLen; start++) { + for (let len = minLen; len <= maxLen; len++) { + if (start + len > haystack.length) break + const window = haystack.slice(start, start + len) + if (lev(window, needle) <= tolerance) return true + } + } + return false +} + +function looksLikeCallsignKey(key: string, label?: string): boolean { + const probe = `${key} ${label || ''}`.toLowerCase() + return /\b(callsign|call sign|callup)\b/.test(probe) || /callsign$/.test(key) || /-callsign\b/.test(key) +} + +function mapTranscriptionToFields(transcription: string): { filled: number; total: number } { + if (!activeLesson.value || !scenario.value) return { filled: 0, total: 0 } + const normalized = norm(transcription) + if (!normalized) return { filled: 0, total: activeLesson.value.fields.length } + let filled = 0 + for (const field of activeLesson.value.fields) { + const expectedRaw = (field.expected(scenario.value) || '').trim() + if (!expectedRaw) continue + const altList = field.alternatives ? (field.alternatives(scenario.value) || []) : [] + const candidates = Array.from(new Set([expectedRaw, ...altList] + .map(c => (c || '').trim()) + .filter(Boolean) + .map(norm) + .filter(c => c.length >= 1))) + + const isCallsign = looksLikeCallsignKey(field.key, field.label) + let matched = false + for (const cand of candidates) { + if (!cand) continue + if (normalized.includes(cand)) { matched = true; break } + if (isCallsign && cand.length >= 4 && fuzzyContains(normalized, cand)) { matched = true; break } + } + if (matched) { + userAnswers[field.key] = expectedRaw + filled++ + } + } + return { filled, total: activeLesson.value.fields.length } +} + +async function blobToBase64(blob: Blob): Promise { + const buf = await blob.arrayBuffer() + const bytes = new Uint8Array(buf) + let bin = '' + const chunkSize = 0x8000 + for (let i = 0; i < bytes.length; i += chunkSize) { + bin += String.fromCharCode.apply(null, Array.from(bytes.subarray(i, i + chunkSize))) + } + return btoa(bin) +} + +async function processSTTAudio(blob: Blob) { + if (!activeLesson.value) return + sttTranscribing.value = true + sttError.value = '' + try { + if (!blob.size) { + sttError.value = 'No audio captured' + return + } + const base64 = await blobToBase64(blob) + const result = await api.post<{ success: boolean; transcription: string }>('/api/atc/ptt', { + audio: base64, + moduleId: current.value?.id || 'classroom', + lessonId: activeLesson.value.id, + format: 'webm', + }) + if (result?.success && result.transcription) { + sttLastTranscription.value = result.transcription + const summary = mapTranscriptionToFields(result.transcription) + toast.value = { + show: true, + text: summary.filled + ? `Filled ${summary.filled}/${summary.total} field${summary.total === 1 ? '' : 's'} from your readback` + : 'Transcribed, but couldn’t map any field. Edit manually below.' + } + setTimeout(() => { toast.value.show = false }, 3500) + } else { + sttError.value = 'No speech detected' + } + } catch (err: any) { + console.error('STT failed', err) + const msg = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Transcription failed' + sttError.value = String(msg) + } finally { + sttTranscribing.value = false + } +} + +async function startSTTRecording() { + if (sttRecording.value || sttTranscribing.value) return + if (!sttSupported) { + sttError.value = 'Your browser does not support microphone recording' + return + } + sttError.value = '' + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true } + }) + sttStream.value = stream + const recorder = new MediaRecorder(stream) + sttMediaRecorder.value = recorder + sttChunks.value = [] + recorder.ondataavailable = (e) => { + if (e.data && e.data.size > 0) sttChunks.value.push(e.data) + } + recorder.onstop = async () => { + const blob = new Blob(sttChunks.value, { type: recorder.mimeType || 'audio/webm' }) + sttStream.value?.getTracks().forEach(t => t.stop()) + sttStream.value = null + sttMediaRecorder.value = null + sttChunks.value = [] + await processSTTAudio(blob) + } + recorder.start() + sttRecording.value = true + } catch (err: any) { + console.error('STT start failed', err) + sttError.value = err?.name === 'NotAllowedError' + ? 'Microphone access denied — please allow it in your browser' + : 'Could not start recording' + sttRecording.value = false + sttStream.value?.getTracks().forEach(t => t.stop()) + sttStream.value = null + } +} + +function stopSTTRecording() { + if (!sttRecording.value || !sttMediaRecorder.value) return + try { + sttMediaRecorder.value.stop() + } catch (err) { + console.warn('STT stop failed', err) + } + sttRecording.value = false +} + +function toggleSTTRecording() { + if (sttRecording.value) { + stopSTTRecording() + } else { + startSTTRecording() + } +} + const lessonInfo = computed(() => (activeLesson.value && scenario.value ? activeLesson.value.info(scenario.value) : [])) const lessonReference = computed(() => { @@ -3806,6 +4023,9 @@ function resetAnswers(clearResult = false) { if (clearResult) { result.value = null } + // Clear stale STT feedback so it doesn't bleed into the next attempt. + sttLastTranscription.value = '' + sttError.value = '' } function clearAnswers() { @@ -4728,6 +4948,15 @@ onMounted(() => { } void checkSpeechServerAvailability() }) + +onBeforeUnmount(() => { + if (sttMediaRecorder.value && sttRecording.value) { + try { sttMediaRecorder.value.stop() } catch { /* ignore */ } + } + sttStream.value?.getTracks().forEach(t => t.stop()) + sttStream.value = null + sttMediaRecorder.value = null +})