feat(stt): seed Whisper prompt with expected readback + per-field debug UI

Whisper prompt seeding (per request):
- ptt.post.ts builds the prompt as generic ICAO bias + this state's expected
  readback appended LAST (survives the 224-token truncation), in both raw token
  form and spoken ICAO form via new radioSpeech.speakToken().
- pm.vue passes the expected phrase + active variable values; classroom.vue
  passes the lesson's expected field values.

Per-field readback debug:
- sttMatch.matchTranscriptionToFields returns fields[] (matched/missing + which
  view matched) plus normalized/denormalized transcription views.
- useRadioBackend types readback_report on the transmit response.
- pm.vue renders a "Readback check" panel in the right log rail; classroom.vue
  renders per-field rows under the STT panel.

Radio-pronunciation fixes (radioSpeech.ts):
- callsign expander handles multi-letter suffixes (DLH6RK -> Lufthansa six Romeo
  Kilo).
- toRadioSpeech now expands airports (EDDC -> Echo Delta Delta Charlie).
- bare altitudes >=1000 in a clearance context are spoken ("climb initially
  5000" -> "climb initially five thousand feet"); speeds/headings untouched.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
leubeem
2026-06-17 14:12:54 +02:00
parent 453b04881f
commit b80feb80d6
8 changed files with 373 additions and 18 deletions

View File

@@ -22,6 +22,19 @@ export interface RadioTransmitResponse {
auto_advanced_states?: string[]
/** True when the full chain is done — show the completion screen. */
session_complete?: boolean
/** Per-field readback diagnostic for the pilot state just evaluated. */
readback_report?: ReadbackFieldDetail[]
}
export interface ReadbackFieldDetail {
field: string
expected: string
matched: boolean
/** Which accepted spoken form matched ("two five right", "icao_phonetic"), or null. */
matched_via: string | null
/** All spoken forms that would have matched this field. */
accepted_forms: string[]
note?: string | null
}
export function useRadioBackend() {

View File

@@ -1175,6 +1175,23 @@
</div>
</div>
<!-- Per-field readback debug: what was recognised vs missing. -->
<div v-if="!sttRecording && !sttTranscribing && sttLastReport && sttLastReport.fields.length" class="stt-report">
<div
v-for="f in sttLastReport.fields"
:key="f.key"
class="stt-report-row"
:class="f.matched ? 'is-ok' : 'is-missing'"
>
<v-icon size="13">{{ f.matched ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
<span class="stt-report-field">{{ f.key }}</span>
<span class="stt-report-expected">{{ f.expected || '—' }}</span>
<span v-if="f.matched" class="stt-report-via">recognised ({{ f.view }})</span>
<span v-else class="stt-report-via">not recognised</span>
</div>
<div class="stt-report-folded">folded: {{ sttLastReport.denormalized }}</div>
</div>
<div v-if="sttError" class="stt-error-body">{{ sttError }}</div>
<div v-else-if="sttRecording || sttTranscribing" class="stt-waiting">
@@ -2900,6 +2917,7 @@ const sttLastTranscription = ref('')
const sttEditableTranscription = ref('')
const sttFilledFields = reactive<Record<string, boolean>>({})
const sttLastFillSummary = ref<{ filled: number; total: number } | null>(null)
const sttLastReport = ref<import('~~/shared/utils/sttMatch').SttMatchResult | null>(null)
const sttRecordingSeconds = ref(0)
const sttMediaRecorder = ref<MediaRecorder | null>(null)
const sttChunks = ref<Blob[]>([])
@@ -3561,9 +3579,13 @@ function buildSttFieldDefs(): SttFieldDef[] {
}
function mapTranscriptionToFields(transcription: string): { filled: number; total: number } {
if (!activeLesson.value || !scenario.value) return { filled: 0, total: 0 }
if (!activeLesson.value || !scenario.value) {
sttLastReport.value = null
return { filled: 0, total: 0 }
}
const defs = buildSttFieldDefs()
const result = matchTranscriptionToFields(transcription, defs)
sttLastReport.value = result
// Clear stale mic markers for fields not in this match round
Object.keys(sttFilledFields).forEach(k => { delete sttFilledFields[k] })
for (const [key, value] of Object.entries(result.matches)) {
@@ -3585,6 +3607,7 @@ function clearSttResult() {
sttLastTranscription.value = ''
sttEditableTranscription.value = ''
sttLastFillSummary.value = null
sttLastReport.value = null
sttError.value = ''
Object.keys(sttFilledFields).forEach(k => { delete sttFilledFields[k] })
}
@@ -3610,11 +3633,19 @@ async function processSTTAudio(blob: Blob) {
return
}
const base64 = await blobToBase64(blob)
// Seed Whisper with this lesson's expected field values (raw + alternatives);
// the server expands them to spoken ICAO form and biases recognition.
const sttDefs = buildSttFieldDefs()
const expectedTokens = Array.from(new Set(
sttDefs.flatMap(d => [d.expected, ...(d.alternatives ?? [])]).map(t => (t || '').trim()).filter(Boolean)
))
const expectedPhrase = sttDefs.map(d => d.expected).filter(Boolean).join(', ')
const result = await api.post<{ success: boolean; transcription: string }>('/api/atc/ptt', {
audio: base64,
moduleId: current.value?.id || 'classroom',
lessonId: activeLesson.value.id,
format: 'webm',
expected: { phrase: expectedPhrase || undefined, tokens: expectedTokens },
})
if (result?.success && result.transcription) {
const text = result.transcription.trim()
@@ -7034,6 +7065,31 @@ onBeforeUnmount(() => {
letter-spacing: .02em;
}
.stt-report {
display: flex;
flex-direction: column;
gap: 3px;
margin-top: 6px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11.5px;
}
.stt-report-row {
display: flex;
align-items: center;
gap: 6px;
}
.stt-report-row.is-ok { color: #6ee7a8; }
.stt-report-row.is-missing { color: #fca5a5; }
.stt-report-field { color: var(--t2); min-width: 90px; }
.stt-report-expected { color: var(--text); font-weight: 600; }
.stt-report-via { color: var(--t2); opacity: .8; }
.stt-report-folded {
margin-top: 2px;
color: var(--t2);
opacity: .6;
font-size: 11px;
}
.stt-waiting {
display: inline-flex;
align-items: center;

View File

@@ -739,6 +739,31 @@
<!-- Desktop log rail -->
<aside class="pm-lograil">
<!-- STT readback check: what was recognised vs missing in the last call. -->
<div v-if="lastReadbackReport.length" class="pm-readback-check">
<div class="pm-readback-head">
<v-icon size="15">mdi-magnify-scan</v-icon>
<span>Readback check</span>
</div>
<p v-if="lastReadbackTranscript" class="pm-readback-heard">heard: {{ lastReadbackTranscript }}</p>
<div
v-for="r in lastReadbackReport"
:key="r.field"
class="pm-readback-row"
:class="r.matched ? 'is-ok' : 'is-missing'"
>
<v-icon size="13">{{ r.matched ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
<span class="pm-readback-text">
<span class="pm-readback-field">{{ r.field }}</span>
= <span class="pm-readback-expected">{{ r.expected || '—' }}</span>
<template v-if="r.matched"> {{ r.matched_via }}</template>
<template v-else>
not recognised<span v-if="r.accepted_forms.length" class="pm-readback-forms">
(say: {{ r.accepted_forms.join(' / ') }})</span>
</template>
</span>
</div>
</div>
<CommLog :entries="log" :limit="30" dense @clear="clearLog" />
</aside>
</div>
@@ -1323,12 +1348,16 @@ const backendSessionId = ref<string | null>(null)
const lastControllerSay = ref<string | null>(null)
// Authoritative expected pilot phrase from the backend — replaces local engine rendering.
const backendExpectedPhrase = ref<string | null>(null)
// Per-field readback diagnostic from the last transmission (STT debug panel).
const lastReadbackReport = ref<import('../composables/useRadioBackend').ReadbackFieldDetail[]>([])
const lastReadbackTranscript = ref<string>('')
// Toggle: show radio pronunciation (wun, tree, squawk niner…) vs plain text.
const showRadioPronunciation = ref(false)
function toRadioSpeech(text: string): string {
return normalizeRadioPhrase(text, {
expandCallsigns: true,
expandAirports: true,
airlineMap: DEFAULT_AIRLINE_TELEPHONY,
sidSuffixIcao: true,
})
@@ -2762,6 +2791,10 @@ const handlePilotTransmission = async (message: string, source: 'text' | 'ptt' =
pmLog.debug('moveToSilent ← next_state_id:', response.next_state_id)
moveToSilent(response.next_state_id)
// Capture the per-field readback diagnostic for the STT debug panel.
lastReadbackReport.value = response.readback_report ?? []
lastReadbackTranscript.value = transcript
// Update expected phrase AFTER cursor moves so any local-engine reactive
// updates from moveToSilent have already settled.
backendExpectedPhrase.value = response.expected_pilot_template ?? null
@@ -3306,6 +3339,24 @@ function arrayBufferToBase64(buffer: ArrayBuffer): string {
return btoa(binary)
}
// Build the Whisper-prompt seed for the current state: the expected pilot
// phrase plus the active variable values (callsign, SID, squawk, runway,
// frequencies…). The server expands these to spoken ICAO form and appends them
// to the bias prompt so Whisper is steered toward exactly what's expected.
function buildSttExpected(): { phrase?: string; tokens: string[] } {
const phrase = backendExpectedPhrase.value?.trim() || undefined
const dict = ((vars as any).value ?? {}) as Record<string, unknown>
const tokens: string[] = []
for (const val of Object.values(dict)) {
if (typeof val === 'number') { tokens.push(String(val)); continue }
if (typeof val === 'string') {
const t = val.trim()
if (t && t.length <= 24) tokens.push(t)
}
}
return { phrase, tokens }
}
const processTransmission = async (audioBlob: Blob, isIntercom: boolean, format: 'wav' | 'webm' = 'webm') => {
const channel = isIntercom ? 'INTERCOM' : 'RADIO'
pmLog.info(`PTT ▶ ${channel} blob=${(audioBlob.size / 1024).toFixed(1)}KB fmt=${format} session=${backendSessionId.value?.slice(0,8) ?? 'none'}`)
@@ -3342,6 +3393,7 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean, format:
lessonId: currentState.value?.id || 'general',
format,
sessionId: backendSessionId.value || undefined,
expected: buildSttExpected(),
})
if (result.success) {
@@ -4644,6 +4696,44 @@ onUnmounted(() => {
display: none;
}
.pm-readback-check {
margin-bottom: 10px;
padding: 10px 12px;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 12px;
background: rgba(255, 255, 255, 0.04);
font-size: 11.5px;
}
.pm-readback-head {
display: flex;
align-items: center;
gap: 6px;
text-transform: uppercase;
letter-spacing: .12em;
font-size: 10px;
font-weight: 600;
color: rgba(255, 255, 255, 0.55);
margin-bottom: 6px;
}
.pm-readback-heard {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
color: rgba(255, 255, 255, 0.5);
margin-bottom: 6px;
font-size: 11px;
}
.pm-readback-row {
display: flex;
align-items: flex-start;
gap: 6px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
line-height: 1.5;
}
.pm-readback-row.is-ok { color: #6ee7a8; }
.pm-readback-row.is-missing { color: #fca5a5; }
.pm-readback-field { color: rgba(255, 255, 255, 0.7); }
.pm-readback-expected { color: #fff; font-weight: 600; }
.pm-readback-forms { color: rgba(255, 255, 255, 0.4); }
.pm-bottomnav {
flex: 0 0 auto;
display: flex;