mirror of
https://github.com/OpenSquawk/OpenSquawk
synced 2026-08-08 02:25:36 +08:00
feat(stt): seed Whisper prompt with expected readback + per-field debug UI
Whisper prompt seeding (per request):
- ptt.post.ts builds the prompt as generic ICAO bias + this state's expected
readback appended LAST (survives the 224-token truncation), in both raw token
form and spoken ICAO form via new radioSpeech.speakToken().
- pm.vue passes the expected phrase + active variable values; classroom.vue
passes the lesson's expected field values.
Per-field readback debug:
- sttMatch.matchTranscriptionToFields returns fields[] (matched/missing + which
view matched) plus normalized/denormalized transcription views.
- useRadioBackend types readback_report on the transmit response.
- pm.vue renders a "Readback check" panel in the right log rail; classroom.vue
renders per-field rows under the STT panel.
Radio-pronunciation fixes (radioSpeech.ts):
- callsign expander handles multi-letter suffixes (DLH6RK -> Lufthansa six Romeo
Kilo).
- toRadioSpeech now expands airports (EDDC -> Echo Delta Delta Charlie).
- bare altitudes >=1000 in a clearance context are spoken ("climb initially
5000" -> "climb initially five thousand feet"); speeds/headings untouched.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -22,6 +22,19 @@ export interface RadioTransmitResponse {
|
||||
auto_advanced_states?: string[]
|
||||
/** True when the full chain is done — show the completion screen. */
|
||||
session_complete?: boolean
|
||||
/** Per-field readback diagnostic for the pilot state just evaluated. */
|
||||
readback_report?: ReadbackFieldDetail[]
|
||||
}
|
||||
|
||||
export interface ReadbackFieldDetail {
|
||||
field: string
|
||||
expected: string
|
||||
matched: boolean
|
||||
/** Which accepted spoken form matched ("two five right", "icao_phonetic"), or null. */
|
||||
matched_via: string | null
|
||||
/** All spoken forms that would have matched this field. */
|
||||
accepted_forms: string[]
|
||||
note?: string | null
|
||||
}
|
||||
|
||||
export function useRadioBackend() {
|
||||
|
||||
@@ -1175,6 +1175,23 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Per-field readback debug: what was recognised vs missing. -->
|
||||
<div v-if="!sttRecording && !sttTranscribing && sttLastReport && sttLastReport.fields.length" class="stt-report">
|
||||
<div
|
||||
v-for="f in sttLastReport.fields"
|
||||
:key="f.key"
|
||||
class="stt-report-row"
|
||||
:class="f.matched ? 'is-ok' : 'is-missing'"
|
||||
>
|
||||
<v-icon size="13">{{ f.matched ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
|
||||
<span class="stt-report-field">{{ f.key }}</span>
|
||||
<span class="stt-report-expected">{{ f.expected || '—' }}</span>
|
||||
<span v-if="f.matched" class="stt-report-via">recognised ({{ f.view }})</span>
|
||||
<span v-else class="stt-report-via">not recognised</span>
|
||||
</div>
|
||||
<div class="stt-report-folded">folded: {{ sttLastReport.denormalized }}</div>
|
||||
</div>
|
||||
|
||||
<div v-if="sttError" class="stt-error-body">{{ sttError }}</div>
|
||||
|
||||
<div v-else-if="sttRecording || sttTranscribing" class="stt-waiting">
|
||||
@@ -2900,6 +2917,7 @@ const sttLastTranscription = ref('')
|
||||
const sttEditableTranscription = ref('')
|
||||
const sttFilledFields = reactive<Record<string, boolean>>({})
|
||||
const sttLastFillSummary = ref<{ filled: number; total: number } | null>(null)
|
||||
const sttLastReport = ref<import('~~/shared/utils/sttMatch').SttMatchResult | null>(null)
|
||||
const sttRecordingSeconds = ref(0)
|
||||
const sttMediaRecorder = ref<MediaRecorder | null>(null)
|
||||
const sttChunks = ref<Blob[]>([])
|
||||
@@ -3561,9 +3579,13 @@ function buildSttFieldDefs(): SttFieldDef[] {
|
||||
}
|
||||
|
||||
function mapTranscriptionToFields(transcription: string): { filled: number; total: number } {
|
||||
if (!activeLesson.value || !scenario.value) return { filled: 0, total: 0 }
|
||||
if (!activeLesson.value || !scenario.value) {
|
||||
sttLastReport.value = null
|
||||
return { filled: 0, total: 0 }
|
||||
}
|
||||
const defs = buildSttFieldDefs()
|
||||
const result = matchTranscriptionToFields(transcription, defs)
|
||||
sttLastReport.value = result
|
||||
// Clear stale mic markers for fields not in this match round
|
||||
Object.keys(sttFilledFields).forEach(k => { delete sttFilledFields[k] })
|
||||
for (const [key, value] of Object.entries(result.matches)) {
|
||||
@@ -3585,6 +3607,7 @@ function clearSttResult() {
|
||||
sttLastTranscription.value = ''
|
||||
sttEditableTranscription.value = ''
|
||||
sttLastFillSummary.value = null
|
||||
sttLastReport.value = null
|
||||
sttError.value = ''
|
||||
Object.keys(sttFilledFields).forEach(k => { delete sttFilledFields[k] })
|
||||
}
|
||||
@@ -3610,11 +3633,19 @@ async function processSTTAudio(blob: Blob) {
|
||||
return
|
||||
}
|
||||
const base64 = await blobToBase64(blob)
|
||||
// Seed Whisper with this lesson's expected field values (raw + alternatives);
|
||||
// the server expands them to spoken ICAO form and biases recognition.
|
||||
const sttDefs = buildSttFieldDefs()
|
||||
const expectedTokens = Array.from(new Set(
|
||||
sttDefs.flatMap(d => [d.expected, ...(d.alternatives ?? [])]).map(t => (t || '').trim()).filter(Boolean)
|
||||
))
|
||||
const expectedPhrase = sttDefs.map(d => d.expected).filter(Boolean).join(', ')
|
||||
const result = await api.post<{ success: boolean; transcription: string }>('/api/atc/ptt', {
|
||||
audio: base64,
|
||||
moduleId: current.value?.id || 'classroom',
|
||||
lessonId: activeLesson.value.id,
|
||||
format: 'webm',
|
||||
expected: { phrase: expectedPhrase || undefined, tokens: expectedTokens },
|
||||
})
|
||||
if (result?.success && result.transcription) {
|
||||
const text = result.transcription.trim()
|
||||
@@ -7034,6 +7065,31 @@ onBeforeUnmount(() => {
|
||||
letter-spacing: .02em;
|
||||
}
|
||||
|
||||
.stt-report {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
margin-top: 6px;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: 11.5px;
|
||||
}
|
||||
.stt-report-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
.stt-report-row.is-ok { color: #6ee7a8; }
|
||||
.stt-report-row.is-missing { color: #fca5a5; }
|
||||
.stt-report-field { color: var(--t2); min-width: 90px; }
|
||||
.stt-report-expected { color: var(--text); font-weight: 600; }
|
||||
.stt-report-via { color: var(--t2); opacity: .8; }
|
||||
.stt-report-folded {
|
||||
margin-top: 2px;
|
||||
color: var(--t2);
|
||||
opacity: .6;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.stt-waiting {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
|
||||
@@ -739,6 +739,31 @@
|
||||
|
||||
<!-- Desktop log rail -->
|
||||
<aside class="pm-lograil">
|
||||
<!-- STT readback check: what was recognised vs missing in the last call. -->
|
||||
<div v-if="lastReadbackReport.length" class="pm-readback-check">
|
||||
<div class="pm-readback-head">
|
||||
<v-icon size="15">mdi-magnify-scan</v-icon>
|
||||
<span>Readback check</span>
|
||||
</div>
|
||||
<p v-if="lastReadbackTranscript" class="pm-readback-heard">heard: “{{ lastReadbackTranscript }}”</p>
|
||||
<div
|
||||
v-for="r in lastReadbackReport"
|
||||
:key="r.field"
|
||||
class="pm-readback-row"
|
||||
:class="r.matched ? 'is-ok' : 'is-missing'"
|
||||
>
|
||||
<v-icon size="13">{{ r.matched ? 'mdi-check-circle' : 'mdi-close-circle' }}</v-icon>
|
||||
<span class="pm-readback-text">
|
||||
<span class="pm-readback-field">{{ r.field }}</span>
|
||||
= <span class="pm-readback-expected">{{ r.expected || '—' }}</span>
|
||||
<template v-if="r.matched"> → “{{ r.matched_via }}”</template>
|
||||
<template v-else>
|
||||
→ not recognised<span v-if="r.accepted_forms.length" class="pm-readback-forms">
|
||||
(say: {{ r.accepted_forms.join(' / ') }})</span>
|
||||
</template>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<CommLog :entries="log" :limit="30" dense @clear="clearLog" />
|
||||
</aside>
|
||||
</div>
|
||||
@@ -1323,12 +1348,16 @@ const backendSessionId = ref<string | null>(null)
|
||||
const lastControllerSay = ref<string | null>(null)
|
||||
// Authoritative expected pilot phrase from the backend — replaces local engine rendering.
|
||||
const backendExpectedPhrase = ref<string | null>(null)
|
||||
// Per-field readback diagnostic from the last transmission (STT debug panel).
|
||||
const lastReadbackReport = ref<import('../composables/useRadioBackend').ReadbackFieldDetail[]>([])
|
||||
const lastReadbackTranscript = ref<string>('')
|
||||
// Toggle: show radio pronunciation (wun, tree, squawk niner…) vs plain text.
|
||||
const showRadioPronunciation = ref(false)
|
||||
|
||||
function toRadioSpeech(text: string): string {
|
||||
return normalizeRadioPhrase(text, {
|
||||
expandCallsigns: true,
|
||||
expandAirports: true,
|
||||
airlineMap: DEFAULT_AIRLINE_TELEPHONY,
|
||||
sidSuffixIcao: true,
|
||||
})
|
||||
@@ -2762,6 +2791,10 @@ const handlePilotTransmission = async (message: string, source: 'text' | 'ptt' =
|
||||
pmLog.debug('moveToSilent ← next_state_id:', response.next_state_id)
|
||||
moveToSilent(response.next_state_id)
|
||||
|
||||
// Capture the per-field readback diagnostic for the STT debug panel.
|
||||
lastReadbackReport.value = response.readback_report ?? []
|
||||
lastReadbackTranscript.value = transcript
|
||||
|
||||
// Update expected phrase AFTER cursor moves so any local-engine reactive
|
||||
// updates from moveToSilent have already settled.
|
||||
backendExpectedPhrase.value = response.expected_pilot_template ?? null
|
||||
@@ -3306,6 +3339,24 @@ function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
return btoa(binary)
|
||||
}
|
||||
|
||||
// Build the Whisper-prompt seed for the current state: the expected pilot
|
||||
// phrase plus the active variable values (callsign, SID, squawk, runway,
|
||||
// frequencies…). The server expands these to spoken ICAO form and appends them
|
||||
// to the bias prompt so Whisper is steered toward exactly what's expected.
|
||||
function buildSttExpected(): { phrase?: string; tokens: string[] } {
|
||||
const phrase = backendExpectedPhrase.value?.trim() || undefined
|
||||
const dict = ((vars as any).value ?? {}) as Record<string, unknown>
|
||||
const tokens: string[] = []
|
||||
for (const val of Object.values(dict)) {
|
||||
if (typeof val === 'number') { tokens.push(String(val)); continue }
|
||||
if (typeof val === 'string') {
|
||||
const t = val.trim()
|
||||
if (t && t.length <= 24) tokens.push(t)
|
||||
}
|
||||
}
|
||||
return { phrase, tokens }
|
||||
}
|
||||
|
||||
const processTransmission = async (audioBlob: Blob, isIntercom: boolean, format: 'wav' | 'webm' = 'webm') => {
|
||||
const channel = isIntercom ? 'INTERCOM' : 'RADIO'
|
||||
pmLog.info(`PTT ▶ ${channel} blob=${(audioBlob.size / 1024).toFixed(1)}KB fmt=${format} session=${backendSessionId.value?.slice(0,8) ?? 'none'}`)
|
||||
@@ -3342,6 +3393,7 @@ const processTransmission = async (audioBlob: Blob, isIntercom: boolean, format:
|
||||
lessonId: currentState.value?.id || 'general',
|
||||
format,
|
||||
sessionId: backendSessionId.value || undefined,
|
||||
expected: buildSttExpected(),
|
||||
})
|
||||
|
||||
if (result.success) {
|
||||
@@ -4644,6 +4696,44 @@ onUnmounted(() => {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.pm-readback-check {
|
||||
margin-bottom: 10px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
border-radius: 12px;
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
font-size: 11.5px;
|
||||
}
|
||||
.pm-readback-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .12em;
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
color: rgba(255, 255, 255, 0.55);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.pm-readback-heard {
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
color: rgba(255, 255, 255, 0.5);
|
||||
margin-bottom: 6px;
|
||||
font-size: 11px;
|
||||
}
|
||||
.pm-readback-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 6px;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.pm-readback-row.is-ok { color: #6ee7a8; }
|
||||
.pm-readback-row.is-missing { color: #fca5a5; }
|
||||
.pm-readback-field { color: rgba(255, 255, 255, 0.7); }
|
||||
.pm-readback-expected { color: #fff; font-weight: 600; }
|
||||
.pm-readback-forms { color: rgba(255, 255, 255, 0.4); }
|
||||
|
||||
.pm-bottomnav {
|
||||
flex: 0 0 auto;
|
||||
display: flex;
|
||||
|
||||
Reference in New Issue
Block a user