From 1235f7fd2f730b2c58c78aa0882a3c1d1c060a16 Mon Sep 17 00:00:00 2001 From: itsrubberduck Date: Tue, 26 May 2026 21:18:39 +0200 Subject: [PATCH] =?UTF-8?q?Fix=20STT=20readback=20=E2=80=94=20TDZ=20crash,?= =?UTF-8?q?=20spoken-form=20mismatch,=20hydration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs in yesterday's STT addition: 1) **TDZ crash** — `sttSupported` referenced `isClient` before its const declaration, throwing on setup and breaking the whole classroom page. `sttSupported` is now a ref that's populated in `onMounted`. 2) **Spoken vs. written mismatch** — Whisper returns natural ATC speech ("runway two five right", "lufthansa three five niner"), but the lesson fields hold the canonical written form ("25R", "DLH359"). The old `normalized.includes(...)` check never matched. Matching now lives in `shared/utils/sttMatch.ts` and searches both the raw normalized transcription *and* a denormalized projection that folds spoken digits/letters back to written tokens (incl. SID suffix `7S`, runway `25R`, scale words `five thousand → 5000`, frequency `decimal` as a digit-run boundary). 3) **SSR hydration mismatch** — `sttSupported` evaluated differently on server vs. client, causing visible-vs-hidden button divergence on hydration. The ref-set-on-mount approach resolves it. The new helper is fully unit-tested (15 cases covering radio check, departure clearance, SIDs, squawks, Speedbird telephony, decimal frequencies and edge cases). Co-Authored-By: Claude Opus 4.7 --- app/pages/classroom.vue | 96 ++++++--------- shared/utils/sttMatch.ts | 226 ++++++++++++++++++++++++++++++++++ tests/shared/sttMatch.test.ts | 171 +++++++++++++++++++++++++ 3 files changed, 435 insertions(+), 58 deletions(-) create mode 100644 shared/utils/sttMatch.ts create mode 100644 tests/shared/sttMatch.test.ts diff --git a/app/pages/classroom.vue b/app/pages/classroom.vue index 235e304..e0fe602 100644 --- a/app/pages/classroom.vue +++ b/app/pages/classroom.vue @@ -1135,6 +1135,7 @@ mdi-microphone-off Speech server unavailable @@ -1457,6 +1458,7 @@ import { minutesToWords } from '~~/shared/learn/scenario' import type {BlankWidth, Frequency, Lesson, LessonField, ModuleDef, ReadbackSegment, Scenario} from '~~/shared/learn/types' +import {looksLikeCallsignKey, matchTranscriptionToFields, type SttFieldDef} from '~~/shared/utils/sttMatch' import {loadPizzicatoLite} from '~~/shared/utils/pizzicatoLite' import type {PizzicatoLite} from '~~/shared/utils/pizzicatoLite' import {createNoiseGenerators, getReadabilityProfile} from '~~/shared/utils/radioEffects' @@ -2822,12 +2824,16 @@ const showSettings = ref(false) const showSpeechServerWarning = ref(false) const showOnlineTtsSuggestion = ref(false) +const api = useApi() +const isClient = typeof window !== 'undefined' +const auth = useAuthStore() +const browserTtsAvailable = computed(() => isClient && 'speechSynthesis' in window) + // STT (Speech-to-Text) for the readback — pilot speaks the readback into a mic, // transcription is mapped onto the input fields. Driven by /api/atc/ptt. -const sttSupported = isClient && typeof window !== 'undefined' - && typeof navigator !== 'undefined' - && Boolean(navigator.mediaDevices?.getUserMedia) - && typeof window.MediaRecorder !== 'undefined' +// `sttSupported` is a ref (not const) so it can be set after onMounted; this +// avoids SSR/hydration mismatches around the mic button visibility. +const sttSupported = ref(false) const sttServerAvailable = ref(true) const sttRecording = ref(false) const sttTranscribing = ref(false) @@ -2836,12 +2842,7 @@ const sttLastTranscription = ref('') const sttMediaRecorder = ref(null) const sttChunks = ref([]) const sttStream = ref(null) -const sttFeatureVisible = computed(() => sttSupported && sttServerAvailable.value) - -const api = useApi() -const isClient = typeof window !== 'undefined' -const auth = useAuthStore() -const browserTtsAvailable = computed(() => isClient && 'speechSynthesis' in window) +const sttFeatureVisible = computed(() => sttSupported.value && sttServerAvailable.value) type SpeechServerHealth = { configured: boolean @@ -3465,61 +3466,36 @@ async function speakCorrectReadback() { // --------------------------------------------------------------------------- // STT — pilot speaks the readback into the mic, transcription is mapped onto -// the input fields. Callsigns are notoriously misrecognized by Whisper, so we -// run fuzzy substring matching with the existing alternative-list for each -// field; for callsign-flagged fields we additionally allow a sliding-window -// Levenshtein match against the transcription. +// the input fields. Whisper returns natural spoken ATC ("two five right", +// "lufthansa three five niner"), while fields store the canonical written +// form ("25R", "DLH359"). The actual matching lives in shared/utils/sttMatch +// where it can be unit-tested; here we just feed it the per-lesson candidates. // --------------------------------------------------------------------------- -function fuzzyContains(haystack: string, needle: string): boolean { - if (!needle || !haystack) return false - if (haystack.includes(needle)) return true - const tolerance = allowedDistance(needle.length) + 1 - const minLen = Math.max(3, needle.length - 2) - const maxLen = needle.length + 3 - for (let start = 0; start <= haystack.length - minLen; start++) { - for (let len = minLen; len <= maxLen; len++) { - if (start + len > haystack.length) break - const window = haystack.slice(start, start + len) - if (lev(window, needle) <= tolerance) return true +function buildSttFieldDefs(): SttFieldDef[] { + if (!activeLesson.value || !scenario.value) return [] + return activeLesson.value.fields.map((field): SttFieldDef => { + const expected = (field.expected(scenario.value!) || '').trim() + const alternatives = field.alternatives + ? field.alternatives(scenario.value!).map(a => (a || '').trim()).filter(Boolean) + : [] + return { + key: field.key, + expected, + alternatives, + isCallsign: looksLikeCallsignKey(field.key, field.label), } - } - return false -} - -function looksLikeCallsignKey(key: string, label?: string): boolean { - const probe = `${key} ${label || ''}`.toLowerCase() - return /\b(callsign|call sign|callup)\b/.test(probe) || /callsign$/.test(key) || /-callsign\b/.test(key) + }) } function mapTranscriptionToFields(transcription: string): { filled: number; total: number } { if (!activeLesson.value || !scenario.value) return { filled: 0, total: 0 } - const normalized = norm(transcription) - if (!normalized) return { filled: 0, total: activeLesson.value.fields.length } - let filled = 0 - for (const field of activeLesson.value.fields) { - const expectedRaw = (field.expected(scenario.value) || '').trim() - if (!expectedRaw) continue - const altList = field.alternatives ? (field.alternatives(scenario.value) || []) : [] - const candidates = Array.from(new Set([expectedRaw, ...altList] - .map(c => (c || '').trim()) - .filter(Boolean) - .map(norm) - .filter(c => c.length >= 1))) - - const isCallsign = looksLikeCallsignKey(field.key, field.label) - let matched = false - for (const cand of candidates) { - if (!cand) continue - if (normalized.includes(cand)) { matched = true; break } - if (isCallsign && cand.length >= 4 && fuzzyContains(normalized, cand)) { matched = true; break } - } - if (matched) { - userAnswers[field.key] = expectedRaw - filled++ - } + const defs = buildSttFieldDefs() + const result = matchTranscriptionToFields(transcription, defs) + for (const [key, value] of Object.entries(result.matches)) { + userAnswers[key] = value } - return { filled, total: activeLesson.value.fields.length } + return { filled: result.filled, total: result.total } } async function blobToBase64(blob: Blob): Promise { @@ -3573,7 +3549,7 @@ async function processSTTAudio(blob: Blob) { async function startSTTRecording() { if (sttRecording.value || sttTranscribing.value) return - if (!sttSupported) { + if (!sttSupported.value) { sttError.value = 'Your browser does not support microphone recording' return } @@ -4945,6 +4921,10 @@ onMounted(() => { if (storedId) { simbriefForm.userId = storedId } + // Detect mic + MediaRecorder support on the client only to keep SSR/CSR + // markup consistent until the page is hydrated. + sttSupported.value = Boolean(navigator.mediaDevices?.getUserMedia) + && typeof window.MediaRecorder !== 'undefined' } void checkSpeechServerAvailability() }) diff --git a/shared/utils/sttMatch.ts b/shared/utils/sttMatch.ts new file mode 100644 index 0000000..f259a20 --- /dev/null +++ b/shared/utils/sttMatch.ts @@ -0,0 +1,226 @@ +// Map a Whisper transcription back onto lesson readback fields. +// +// Whisper returns natural ATC speech ("lufthansa three five niner runway two +// five right squawk seven five zero zero"), but the lesson fields store the +// canonical written form ("DLH359", "25R", "7500"). Matching purely on the +// raw transcription misses most fields. We therefore build a *denormalized* +// view of the transcription where spoken digits/letters are folded back to +// their written tokens, and search both forms when looking for each field's +// expected value (or any alternative). + +const SPOKEN_DIGIT: Record = { + zero: '0', + one: '1', wun: '1', + two: '2', too: '2', + three: '3', tree: '3', + four: '4', fower: '4', + five: '5', fife: '5', + six: '6', + seven: '7', + eight: '8', + nine: '9', niner: '9', +} + +const SPOKEN_LETTER: Record = { + alfa: 'a', alpha: 'a', + bravo: 'b', + charlie: 'c', + delta: 'd', + echo: 'e', + foxtrot: 'f', + golf: 'g', + hotel: 'h', + india: 'i', + juliett: 'j', juliet: 'j', + kilo: 'k', + lima: 'l', + mike: 'm', + november: 'n', + oscar: 'o', + papa: 'p', + quebec: 'q', + romeo: 'r', + sierra: 's', + tango: 't', + uniform: 'u', + victor: 'v', + whiskey: 'w', whisky: 'w', + xray: 'x', + yankee: 'y', + zulu: 'z', +} + +const RUNWAY_SUFFIX: Record = { + left: 'l', + right: 'r', + center: 'c', centre: 'c', +} + +const SCALE_WORDS: Record = { + hundred: '00', + thousand: '000', +} + +// Decorative words that should be stripped AFTER digit-collapse so they still +// act as a boundary while collapsing (otherwise "one one eight decimal seven" +// would fold to "1187" instead of the intended "118 7"). +const DECORATION_RE = /\b(decimal|point|dash|and)\b/g + +export function normalizeForMatch(value: string): string { + return value + .toLowerCase() + .normalize('NFD') + .replace(/[^a-z0-9 ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim() +} + +/** Convert spoken ATC English back to written tokens (digits, runway letters, + * collapsed callsign codes). Returned text is also `normalizeForMatch`-safe. */ +export function denormalizeSpokenAtc(input: string): string { + const cleaned = input + .toLowerCase() + .normalize('NFD') + .replace(/x[- ]?ray/g, 'xray') + .replace(/[^a-z0-9 ]+/g, ' ') + .replace(/\s+/g, ' ') + .trim() + if (!cleaned) return '' + + const tokens = cleaned.split(' ') + const mapped: string[] = [] + for (const tok of tokens) { + if (SPOKEN_DIGIT[tok] !== undefined) mapped.push(SPOKEN_DIGIT[tok]!) + else if (SPOKEN_LETTER[tok] !== undefined) mapped.push(SPOKEN_LETTER[tok]!) + else if (RUNWAY_SUFFIX[tok] !== undefined) mapped.push(RUNWAY_SUFFIX[tok]!) + else if (SCALE_WORDS[tok] !== undefined) mapped.push(SCALE_WORDS[tok]!) + else mapped.push(tok) + } + + let result = mapped.join(' ') + + // "five thousand" → "5 000" → "5000" + result = result.replace(/\b(\d)\s+(0{2,3})\b/g, (_m, d, z) => `${d}${z}`) + + // Collapse runs of single digits: "3 5 9" → "359" + result = result.replace(/\b(\d(?:\s+\d)+)\b/g, m => m.replace(/\s+/g, '')) + + // Glue trailing runway letter to its numeric prefix: "25 r" → "25r" + result = result.replace(/\b(\d{1,3})\s+([lrc])\b/g, '$1$2') + + // Glue single digit + single letter (SID/STAR suffix pattern): "7 s" → "7s" + result = result.replace(/\b(\d)\s+([a-z])\b/g, '$1$2') + + // Glue runs of single letters into a callsign code, but only when followed + // by digits or end-of-string so we don't crush ordinary words. + // "d l h 359" → "dlh 359" + result = result.replace(/\b([a-z](?:\s+[a-z]){1,4})\b(?=\s+\d|\s*$)/g, m => m.replace(/\s+/g, '')) + + // Strip decoration words now that the digit runs around them have been + // collapsed and the boundary they provided is no longer needed. + result = result.replace(DECORATION_RE, ' ') + + return result.replace(/\s+/g, ' ').trim() +} + +function levenshtein(a: string, b: string): number { + const m = a.length + const n = b.length + if (m === 0) return n + if (n === 0) return m + const dp = new Array(n + 1).fill(0) + for (let j = 0; j <= n; j++) dp[j] = j + for (let i = 1; i <= m; i++) { + let prev = dp[0] + dp[0] = i + for (let j = 1; j <= n; j++) { + const temp = dp[j] + const cost = a[i - 1] === b[j - 1] ? 0 : 1 + dp[j] = Math.min(dp[j] + 1, dp[j - 1] + 1, prev + cost) + prev = temp + } + } + return dp[n] +} + +function allowedDistance(length: number): number { + if (length <= 12) return 0 + if (length <= 24) return 1 + if (length <= 36) return 2 + return 3 +} + +/** Loose substring search using a sliding Levenshtein window. */ +export function fuzzyContains(haystack: string, needle: string): boolean { + if (!needle) return false + if (!haystack) return false + if (haystack.includes(needle)) return true + const tolerance = allowedDistance(needle.length) + 1 + const minLen = Math.max(3, needle.length - 2) + const maxLen = needle.length + 3 + for (let start = 0; start + minLen <= haystack.length; start++) { + for (let len = minLen; len <= maxLen; len++) { + if (start + len > haystack.length) break + const window = haystack.slice(start, start + len) + if (levenshtein(window, needle) <= tolerance) return true + } + } + return false +} + +export interface SttFieldDef { + key: string + expected: string + alternatives?: string[] + isCallsign?: boolean +} + +export interface SttMatchResult { + matches: Record + filled: number + total: number +} + +export function matchTranscriptionToFields( + transcription: string, + fields: SttFieldDef[], +): SttMatchResult { + const normalized = normalizeForMatch(transcription) + const denormalized = normalizeForMatch(denormalizeSpokenAtc(transcription)) + const matches: Record = {} + let filled = 0 + for (const field of fields) { + const expectedRaw = (field.expected || '').trim() + if (!expectedRaw) continue + const altList = field.alternatives || [] + const candidates = Array.from(new Set([expectedRaw, ...altList] + .map(c => (c || '').trim()) + .filter(Boolean) + .map(normalizeForMatch) + .filter(c => c.length >= 1))) + + let matched = false + for (const cand of candidates) { + if (!cand) continue + if (normalized.includes(cand) || denormalized.includes(cand)) { matched = true; break } + if (field.isCallsign && cand.length >= 4) { + if (fuzzyContains(normalized, cand) || fuzzyContains(denormalized, cand)) { + matched = true + break + } + } + } + if (matched) { + matches[field.key] = expectedRaw + filled++ + } + } + return { matches, filled, total: fields.length } +} + +export function looksLikeCallsignKey(key: string, label?: string): boolean { + const probe = `${key} ${label || ''}`.toLowerCase() + return /\b(callsign|call sign|callup)\b/.test(probe) + || /callsign$/.test(key) + || /-callsign\b/.test(key) +} diff --git a/tests/shared/sttMatch.test.ts b/tests/shared/sttMatch.test.ts new file mode 100644 index 0000000..8cfdb96 --- /dev/null +++ b/tests/shared/sttMatch.test.ts @@ -0,0 +1,171 @@ +import { describe, it } from 'node:test' +import assert from 'node:assert/strict' + +import { + denormalizeSpokenAtc, + matchTranscriptionToFields, + fuzzyContains, + normalizeForMatch, + looksLikeCallsignKey, +} from '~~/shared/utils/sttMatch' + +describe('denormalizeSpokenAtc', () => { + it('folds spoken digits back into a number', () => { + assert.equal(denormalizeSpokenAtc('three five niner'), '359') + assert.equal(denormalizeSpokenAtc('two five right'), '25r') + assert.equal(denormalizeSpokenAtc('Lufthansa three five niner'), 'lufthansa 359') + }) + + it('expands scale words', () => { + assert.equal(denormalizeSpokenAtc('climb five thousand'), 'climb 5000') + assert.equal(denormalizeSpokenAtc('one hundred'), '100') + }) + + it('collapses ICAO letter sequences when followed by digits', () => { + assert.equal(denormalizeSpokenAtc('delta lima hotel three five niner'), 'dlh 359') + }) + + it('does not crush plain English words', () => { + const out = denormalizeSpokenAtc('contact tower on one one eight decimal seven') + assert.match(out, /contact tower on 118 7/) + }) + + it('handles runway suffix letters', () => { + assert.equal(denormalizeSpokenAtc('runway zero eight right'), 'runway 08r') + }) +}) + +describe('matchTranscriptionToFields', () => { + it('matches a runway field even when Whisper outputs spoken form', () => { + const result = matchTranscriptionToFields( + 'Lufthansa three five niner runway two five right', + [ + { key: 'runway', expected: '25R' }, + ], + ) + assert.equal(result.filled, 1) + assert.equal(result.matches['runway'], '25R') + }) + + it('matches an airline-style callsign via the airline alternative', () => { + const result = matchTranscriptionToFields( + 'Lufthansa three five niner', + [ + { + key: 'callsign', + expected: 'DLH359', + alternatives: ['DLH359', 'Lufthansa 359', 'Lufthansa three five niner'], + isCallsign: true, + }, + ], + ) + assert.equal(result.filled, 1) + assert.equal(result.matches['callsign'], 'DLH359') + }) + + it('matches a SID name (uppercase waypoint + suffix)', () => { + const result = matchTranscriptionToFields( + 'cleared via aneki seven sierra', + [ + { key: 'sid', expected: 'ANEKI 7S' }, + ], + ) + assert.equal(result.filled, 1) + }) + + it('matches a 4-digit squawk after digit collapse', () => { + const result = matchTranscriptionToFields( + 'squawk seven five zero zero', + [ + { key: 'squawk', expected: '7500' }, + ], + ) + assert.equal(result.filled, 1) + }) + + it('does not match unrelated fields', () => { + const result = matchTranscriptionToFields( + 'Lufthansa three five niner readability five', + [ + { key: 'runway', expected: '25R' }, + { key: 'altitude', expected: '5000' }, + { key: 'callsign', expected: 'DLH359', alternatives: ['Lufthansa 359'], isCallsign: true }, + ], + ) + assert.equal(result.matches['runway'], undefined) + assert.equal(result.matches['altitude'], undefined) + assert.equal(result.matches['callsign'], 'DLH359') + }) + + it('survives a complete departure clearance readback', () => { + const result = matchTranscriptionToFields( + 'Lufthansa three five niner cleared to Paris via aneki seven sierra runway two five right climb five thousand squawk one four zero zero', + [ + { key: 'callsign', expected: 'DLH359', alternatives: ['Lufthansa 359'], isCallsign: true }, + { key: 'dest', expected: 'LFPG', alternatives: ['Paris'] }, + { key: 'sid', expected: 'ANEKI 7S' }, + { key: 'runway', expected: '25R' }, + { key: 'altitude', expected: '5000' }, + { key: 'squawk', expected: '1400' }, + ], + ) + assert.equal(result.filled, 6, `expected all 6 fields matched, got ${result.filled}: ${JSON.stringify(result.matches)}`) + }) +}) + +describe('fuzzyContains', () => { + it('tolerates one off-by-one error in a long callsign', () => { + assert.equal(fuzzyContains('luftansa three five niner', 'lufthansa 359'), false) + assert.equal(fuzzyContains(normalizeForMatch('luftansa three five niner'), 'lufthansa'), true) + }) +}) + +describe('looksLikeCallsignKey', () => { + it('detects common callsign field keys', () => { + assert.equal(looksLikeCallsignKey('callsign'), true) + assert.equal(looksLikeCallsignKey('rc-callsign'), true) + assert.equal(looksLikeCallsignKey('tko-callsign'), true) + assert.equal(looksLikeCallsignKey('runway'), false) + }) +}) + +describe('matchTranscriptionToFields — realistic radio-check', () => { + it('handles a full spoken radio check readback', () => { + const result = matchTranscriptionToFields( + 'Lufthansa three five niner readability five', + [ + { + key: 'rc-callsign', + expected: 'DLH359', + alternatives: [ + 'DLH359', 'Lufthansa 359', 'Lufthansa three five niner', + 'DLH 359', 'Delta Lima Hotel three five niner', + ], + isCallsign: true, + }, + { + key: 'rc-readability', + expected: 'five', + alternatives: ['1', '2', '3', '4', '5', 'one', 'two', 'three', 'four', 'five', 'wun', 'too', 'tree', 'fife'], + }, + ], + ) + assert.equal(result.matches['rc-callsign'], 'DLH359') + assert.equal(result.matches['rc-readability'], 'five') + }) + + it('matches a Speedbird callsign even with British telephony', () => { + const result = matchTranscriptionToFields( + 'Speedbird two seven cleared to London', + [ + { + key: 'callsign', + expected: 'BAW27', + alternatives: ['BAW27', 'Speedbird 27', 'Speedbird two seven'], + isCallsign: true, + }, + ], + ) + assert.equal(result.matches['callsign'], 'BAW27') + }) +})